Move literally everything out of src/or

This commit won't build yet -- it just puts everything in a slightly
more logical place.

The reasoning here is that "src/core" will hold the stuff that every (or
nearly every) tor instance will need in order to do onion routing.
Other features (including some necessary ones) will live in
"src/feature".  The "src/app" directory will hold the stuff needed
to have Tor be an application you can actually run.

This commit DOES NOT refactor the former contents of src/or into a
logical set of acyclic libraries, or change any code at all.  That
will have to come in the future.

We will continue to move things around and split them in the future,
but I hope this lays a reasonable groundwork for doing so.

140 files changed, 106275 insertions, 0 deletions

diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c
new file mode 100644
index 0000000000..efedf2dc78
--- /dev/null
+++ b/src/feature/api/tor_api.c
@@ -0,0 +1,88 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_api.c
+ **/
+
+#include "or/tor_api.h"
+#include "or/tor_api_internal.h"
+
+// Include this after the above headers, to insure that they don't
+// depend on anything else.
+#include "orconfig.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// We don't want to use tor_malloc and tor_free here, since this needs
+// to run before anything is initialized at all, and ought to run when
+// we're not linked to anything at all.
+
+#define raw_malloc malloc
+#define raw_free free
+
+tor_main_configuration_t *
+tor_main_configuration_new(void)
+{
+  static const char *fake_argv[] = { "tor" };
+  tor_main_configuration_t *cfg = raw_malloc(sizeof(*cfg));
+  if (cfg == NULL)
+    return NULL;
+
+  memset(cfg, 0, sizeof(*cfg));
+
+  cfg->argc = 1;
+  cfg->argv = (char **) fake_argv;
+
+  return cfg;
+}
+
+int
+tor_main_configuration_set_command_line(tor_main_configuration_t *cfg,
+                                        int argc, char *argv[])
+{
+  if (cfg == NULL)
+    return -1;
+  cfg->argc = argc;
+  cfg->argv = argv;
+  return 0;
+}
+
+void
+tor_main_configuration_free(tor_main_configuration_t *cfg)
+{
+  if (cfg == NULL)
+    return;
+  raw_free(cfg);
+}
+
+/* Main entry point for the Tor process.  Called from main().
+ *
+ * This function is distinct from main() only so we can link main.c into
+ * the unittest binary without conflicting with the unittests' main.
+ *
+ * Some embedders have historically called this function; but that usage is
+ * deprecated: they should use tor_run_main() instead.
+ */
+int
+tor_main(int argc, char *argv[])
+{
+  tor_main_configuration_t *cfg = tor_main_configuration_new();
+  if (!cfg) {
+    puts("INTERNAL ERROR: Allocation failure. Cannot proceed");
+    return 1;
+  }
+  if (tor_main_configuration_set_command_line(cfg, argc, argv) < 0) {
+    puts("INTERNAL ERROR: Can't set command line. Cannot proceed.");
+    return 1;
+  }
+  int rv = tor_run_main(cfg);
+  tor_main_configuration_free(cfg);
+  return rv;
+}
+
diff --git a/src/feature/api/tor_api.h b/src/feature/api/tor_api.h
new file mode 100644
index 0000000000..ead9493c1f
--- /dev/null
+++ b/src/feature/api/tor_api.h
@@ -0,0 +1,101 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_api.h
+ * \brief Public C API for the Tor network service.
+ *
+ * This interface is intended for use by programs that need to link Tor as
+ * a library, and launch it in a separate thread.  If you have the ability
+ * to run Tor as a separate executable, you should probably do that instead
+ * of embedding it as a library.
+ *
+ * To use this API, first construct a tor_main_configuration_t object using
+ * tor_main_configuration_new().  Then, you use one or more other function
+ * calls (such as tor_main_configuration_set_command_line() to configure how
+ * Tor should be run.  Finally, you pass the configuration object to
+ * tor_run_main().
+ *
+ * At this point, tor_run_main() will block its thread to run a Tor daemon;
+ * when the Tor daemon exits, it will return.  See notes on bugs and
+ * limitations below.
+ *
+ * There is no other public C API to Tor: calling any C Tor function not
+ * documented in this file is not guaranteed to be stable.
+ **/
+
+#ifndef TOR_API_H
+#define TOR_API_H
+
+typedef struct tor_main_configuration_t tor_main_configuration_t;
+
+/**
+ * Create and return a new tor_main_configuration().
+ */
+tor_main_configuration_t *tor_main_configuration_new(void);
+
+/**
+ * Set the command-line arguments in <b>cfg</b>.
+ *
+ * The <b>argc</b> and <b>argv</b> values here are as for main().  The
+ * contents of the argv pointer must remain unchanged until tor_run_main() has
+ * finished and you call tor_main_configuration_free().
+ *
+ * Return 0 on success, -1 on failure.
+ */
+int tor_main_configuration_set_command_line(tor_main_configuration_t *cfg,
+                                            int argc, char *argv[]);
+
+/**
+ * Release all storage held in <b>cfg</b>.
+ *
+ * Once you have passed a tor_main_configuration_t to tor_run_main(), you
+ * must not free it until tor_run_main() has finished.
+ */
+void tor_main_configuration_free(tor_main_configuration_t *cfg);
+
+/**
+ * Run the tor process, as if from the command line.
+ *
+ * The command line arguments from tor_main_configuration_set_command_line()
+ * are taken as if they had been passed to main().
+ *
+ * This function will not return until Tor is done running.  It returns zero
+ * on success, and nonzero on failure.
+ *
+ * If you want to control when Tor exits, make sure to configure a control
+ * socket. The OwningControllerFD option may be helpful there.
+ *
+ * BUG 23847: Sometimes, if you call tor_main a second time (after it has
+ * returned), Tor may crash or behave strangely.  We have fixed all issues of
+ * this type that we could find, but more may remain.
+ *
+ * LIMITATION: You cannot run more than one instance of Tor in the same
+ * process at the same time. Concurrent calls will cause undefined behavior.
+ * We do not currently have plans to change this.
+ *
+ * LIMITATION: While we will try to fix any problems found here, you
+ * should be aware that Tor was originally written to run as its own
+ * process, and that the functionality of this file was added later.  If
+ * you find any bugs or strange behavior, please report them, and we'll
+ * try to straighten them out.
+ */
+int tor_run_main(const tor_main_configuration_t *);
+
+/**
+ * Run the tor process, as if from the command line.
+ *
+ * @deprecated Using this function from outside Tor is deprecated; you should
+ * use tor_run_main() instead.
+ *
+ * BUGS: This function has all the same bugs as tor_run_main().
+ *
+ * LIMITATIONS: This function has all the limitations of tor_run_main().
+ */
+int tor_main(int argc, char **argv);
+
+#endif /* !defined(TOR_API_H) */
+
diff --git a/src/feature/api/tor_api_internal.h b/src/feature/api/tor_api_internal.h
new file mode 100644
index 0000000000..2c392a68de
--- /dev/null
+++ b/src/feature/api/tor_api_internal.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_API_INTERNAL_H
+#define TOR_API_INTERNAL_H
+
+/* The contents of this type are private; don't mess with them from outside
+ * Tor. */
+struct tor_main_configuration_t {
+  /** As in main() */
+  int argc;
+  /** As in main(). This pointer is owned by the caller */
+  char **argv;
+};
+
+#endif /* !defined(TOR_API_INTERNAL_H) */
+
diff --git a/src/feature/client/addressmap.c b/src/feature/client/addressmap.c
new file mode 100644
index 0000000000..ba78a5f908
--- /dev/null
+++ b/src/feature/client/addressmap.c
@@ -0,0 +1,1156 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file addressmap.c
+ *
+ * \brief The addressmap module manages the processes by which we rewrite
+ * addresses in client requess.  It handles the MapAddress controller and
+ * torrc commands, and the TrackHostExits feature, and the client-side DNS
+ * cache (deprecated).
+ */
+
+#define ADDRESSMAP_PRIVATE
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "or/or.h"
+#include "or/addressmap.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "or/connection_edge.h"
+#include "or/control.h"
+#include "or/dns.h"
+#include "or/nodelist.h"
+#include "or/routerset.h"
+
+#include "or/entry_connection_st.h"
+
+/** A client-side struct to remember requests to rewrite addresses
+ * to new addresses. These structs are stored in the hash table
+ * "addressmap" below.
+ *
+ * There are 5 ways to set an address mapping:
+ * - A MapAddress command from the controller [permanent]
+ * - An AddressMap directive in the torrc [permanent]
+ * - When a TrackHostExits torrc directive is triggered [temporary]
+ * - When a DNS resolve succeeds [temporary]
+ * - When a DNS resolve fails [temporary]
+ *
+ * When an addressmap request is made but one is already registered,
+ * the new one is replaced only if the currently registered one has
+ * no "new_address" (that is, it's in the process of DNS resolve),
+ * or if the new one is permanent (expires==0 or 1).
+ *
+ * (We overload the 'expires' field, using "0" for mappings set via
+ * the configuration file, "1" for mappings set from the control
+ * interface, and other values for DNS and TrackHostExit mappings that can
+ * expire.)
+ *
+ * A mapping may be 'wildcarded'.  If "src_wildcard" is true, then
+ * any address that ends with a . followed by the key for this entry will
+ * get remapped by it.  If "dst_wildcard" is also true, then only the
+ * matching suffix of such addresses will get replaced by new_address.
+ */
+typedef struct {
+  char *new_address;
+  time_t expires;
+  addressmap_entry_source_bitfield_t source:3;
+  unsigned src_wildcard:1;
+  unsigned dst_wildcard:1;
+  short num_resolve_failures;
+} addressmap_entry_t;
+
+/** Entry for mapping addresses to which virtual address we mapped them to. */
+typedef struct {
+  char *ipv4_address;
+  char *ipv6_address;
+  char *hostname_address;
+} virtaddress_entry_t;
+
+/** A hash table to store client-side address rewrite instructions. */
+static strmap_t *addressmap=NULL;
+
+/**
+ * Table mapping addresses to which virtual address, if any, we
+ * assigned them to.
+ *
+ * We maintain the following invariant: if [A,B] is in
+ * virtaddress_reversemap, then B must be a virtual address, and [A,B]
+ * must be in addressmap.  We do not require that the converse hold:
+ * if it fails, then we could end up mapping two virtual addresses to
+ * the same address, which is no disaster.
+ **/
+static strmap_t *virtaddress_reversemap=NULL;
+
+/** Initialize addressmap. */
+void
+addressmap_init(void)
+{
+  addressmap = strmap_new();
+  virtaddress_reversemap = strmap_new();
+}
+
+#define addressmap_ent_free(ent)                                        \
+  FREE_AND_NULL(addressmap_entry_t, addressmap_ent_free_, (ent))
+
+/** Free the memory associated with the addressmap entry <b>_ent</b>. */
+static void
+addressmap_ent_free_(addressmap_entry_t *ent)
+{
+  if (!ent)
+    return;
+
+  tor_free(ent->new_address);
+  tor_free(ent);
+}
+
+static void
+addressmap_ent_free_void(void *ent)
+{
+  addressmap_ent_free_(ent);
+}
+
+#define addressmap_virtaddress_ent_free(ent)                            \
+  FREE_AND_NULL(virtaddress_entry_t, addressmap_virtaddress_ent_free_, (ent))
+
+/** Free storage held by a virtaddress_entry_t* entry in <b>_ent</b>. */
+static void
+addressmap_virtaddress_ent_free_(virtaddress_entry_t *ent)
+{
+  if (!ent)
+    return;
+  tor_free(ent->ipv4_address);
+  tor_free(ent->ipv6_address);
+  tor_free(ent->hostname_address);
+  tor_free(ent);
+}
+
+static void
+addressmap_virtaddress_ent_free_void(void *ent)
+{
+  addressmap_virtaddress_ent_free_(ent);
+}
+
+/** Remove <b>address</b> (which must map to <b>ent</b>) from the
+ * virtual address map. */
+static void
+addressmap_virtaddress_remove(const char *address, addressmap_entry_t *ent)
+{
+  if (ent && ent->new_address &&
+      address_is_in_virtual_range(ent->new_address)) {
+    virtaddress_entry_t *ve =
+      strmap_get(virtaddress_reversemap, ent->new_address);
+    /*log_fn(LOG_NOTICE,"remove reverse mapping for %s",ent->new_address);*/
+    if (ve) {
+      if (!strcmp(address, ve->ipv4_address))
+        tor_free(ve->ipv4_address);
+      if (!strcmp(address, ve->ipv6_address))
+        tor_free(ve->ipv6_address);
+      if (!strcmp(address, ve->hostname_address))
+        tor_free(ve->hostname_address);
+      if (!ve->ipv4_address && !ve->ipv6_address && !ve->hostname_address) {
+        tor_free(ve);
+        strmap_remove(virtaddress_reversemap, ent->new_address);
+      }
+    }
+  }
+}
+
+/** Remove <b>ent</b> (which must be mapped to by <b>address</b>) from the
+ * client address maps, and then free it. */
+static void
+addressmap_ent_remove(const char *address, addressmap_entry_t *ent)
+{
+  addressmap_virtaddress_remove(address, ent);
+  addressmap_ent_free(ent);
+}
+
+/** Unregister all TrackHostExits mappings from any address to
+ * *.exitname.exit. */
+void
+clear_trackexithost_mappings(const char *exitname)
+{
+  char *suffix = NULL;
+  if (!addressmap || !exitname)
+    return;
+  tor_asprintf(&suffix, ".%s.exit", exitname);
+  tor_strlower(suffix);
+
+  STRMAP_FOREACH_MODIFY(addressmap, address, addressmap_entry_t *, ent) {
+    if (ent->source == ADDRMAPSRC_TRACKEXIT &&
+        !strcmpend(ent->new_address, suffix)) {
+      addressmap_ent_remove(address, ent);
+      MAP_DEL_CURRENT(address);
+    }
+  } STRMAP_FOREACH_END;
+
+  tor_free(suffix);
+}
+
+/** Remove all TRACKEXIT mappings from the addressmap for which the target
+ * host is unknown or no longer allowed, or for which the source address
+ * is no longer in trackexithosts. */
+void
+addressmap_clear_excluded_trackexithosts(const or_options_t *options)
+{
+  const routerset_t *allow_nodes = options->ExitNodes;
+  const routerset_t *exclude_nodes = options->ExcludeExitNodesUnion_;
+
+  if (!addressmap)
+    return;
+  if (routerset_is_empty(allow_nodes))
+    allow_nodes = NULL;
+  if (allow_nodes == NULL && routerset_is_empty(exclude_nodes))
+    return;
+
+  STRMAP_FOREACH_MODIFY(addressmap, address, addressmap_entry_t *, ent) {
+    size_t len;
+    const char *target = ent->new_address, *dot;
+    char *nodename;
+    const node_t *node;
+
+    if (!target) {
+      /* DNS resolving in progress */
+      continue;
+    } else if (strcmpend(target, ".exit")) {
+      /* Not a .exit mapping */
+      continue;
+    } else if (ent->source != ADDRMAPSRC_TRACKEXIT) {
+      /* Not a trackexit mapping. */
+      continue;
+    }
+    len = strlen(target);
+    if (len < 6)
+      continue; /* malformed. */
+    dot = target + len - 6; /* dot now points to just before .exit */
+    while (dot > target && *dot != '.')
+      dot--;
+    if (*dot == '.') dot++;
+    nodename = tor_strndup(dot, len-5-(dot-target));
+    node = node_get_by_nickname(nodename, NNF_NO_WARN_UNNAMED);
+    tor_free(nodename);
+    if (!node ||
+        (allow_nodes && !routerset_contains_node(allow_nodes, node)) ||
+        routerset_contains_node(exclude_nodes, node) ||
+        !hostname_in_track_host_exits(options, address)) {
+      /* We don't know this one, or we want to be rid of it. */
+      addressmap_ent_remove(address, ent);
+      MAP_DEL_CURRENT(address);
+    }
+  } STRMAP_FOREACH_END;
+}
+
+/** Return true iff <b>address</b> is one that we are configured to
+ * automap on resolve according to <b>options</b>. */
+int
+addressmap_address_should_automap(const char *address,
+                                  const or_options_t *options)
+{
+  const smartlist_t *suffix_list = options->AutomapHostsSuffixes;
+
+  if (!suffix_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(suffix_list, const char *, suffix) {
+    if (!strcmp(suffix, "."))
+      return 1;
+    if (!strcasecmpend(address, suffix))
+      return 1;
+  } SMARTLIST_FOREACH_END(suffix);
+  return 0;
+}
+
+/** Remove all AUTOMAP mappings from the addressmap for which the
+ * source address no longer matches AutomapHostsSuffixes, which is
+ * no longer allowed by AutomapHostsOnResolve, or for which the
+ * target address is no longer in the virtual network. */
+void
+addressmap_clear_invalid_automaps(const or_options_t *options)
+{
+  int clear_all = !options->AutomapHostsOnResolve;
+  const smartlist_t *suffixes = options->AutomapHostsSuffixes;
+
+  if (!addressmap)
+    return;
+
+  if (!suffixes)
+    clear_all = 1; /* This should be impossible, but let's be sure. */
+
+  STRMAP_FOREACH_MODIFY(addressmap, src_address, addressmap_entry_t *, ent) {
+    int remove_this = clear_all;
+    if (ent->source != ADDRMAPSRC_AUTOMAP)
+      continue; /* not an automap mapping. */
+
+    if (!remove_this) {
+      remove_this = ! addressmap_address_should_automap(src_address, options);
+    }
+
+    if (!remove_this && ! address_is_in_virtual_range(ent->new_address))
+      remove_this = 1;
+
+    if (remove_this) {
+      addressmap_ent_remove(src_address, ent);
+      MAP_DEL_CURRENT(src_address);
+    }
+  } STRMAP_FOREACH_END;
+}
+
+/** Remove all entries from the addressmap that were set via the
+ * configuration file or the command line. */
+void
+addressmap_clear_configured(void)
+{
+  addressmap_get_mappings(NULL, 0, 0, 0);
+}
+
+/** Remove all entries from the addressmap that are set to expire, ever. */
+void
+addressmap_clear_transient(void)
+{
+  addressmap_get_mappings(NULL, 2, TIME_MAX, 0);
+}
+
+/** Clean out entries from the addressmap cache that were
+ * added long enough ago that they are no longer valid.
+ */
+void
+addressmap_clean(time_t now)
+{
+  addressmap_get_mappings(NULL, 2, now, 0);
+}
+
+/** Free all the elements in the addressmap, and free the addressmap
+ * itself. */
+void
+addressmap_free_all(void)
+{
+  strmap_free(addressmap, addressmap_ent_free_void);
+  addressmap = NULL;
+
+  strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free_void);
+  virtaddress_reversemap = NULL;
+}
+
+/** Try to find a match for AddressMap expressions that use
+ *  wildcard notation such as '*.c.d *.e.f' (so 'a.c.d' will map to 'a.e.f') or
+ *  '*.c.d a.b.c' (so 'a.c.d' will map to a.b.c).
+ *  Return the matching entry in AddressMap or NULL if no match is found.
+ *  For expressions such as '*.c.d *.e.f', truncate <b>address</b> 'a.c.d'
+ *  to 'a' before we return the matching AddressMap entry.
+ *
+ * This function does not handle the case where a pattern of the form "*.c.d"
+ * matches the address c.d -- that's done by the main addressmap_rewrite
+ * function.
+ */
+static addressmap_entry_t *
+addressmap_match_superdomains(char *address)
+{
+  addressmap_entry_t *val;
+  char *cp;
+
+  cp = address;
+  while ((cp = strchr(cp, '.'))) {
+    /* cp now points to a suffix of address that begins with a . */
+    val = strmap_get_lc(addressmap, cp+1);
+    if (val && val->src_wildcard) {
+      if (val->dst_wildcard)
+        *cp = '\0';
+      return val;
+    }
+    ++cp;
+  }
+  return NULL;
+}
+
+/** Look at address, and rewrite it until it doesn't want any
+ * more rewrites; but don't get into an infinite loop.
+ * Don't write more than maxlen chars into address.  Return true if the
+ * address changed; false otherwise.  Set *<b>expires_out</b> to the
+ * expiry time of the result, or to <b>time_max</b> if the result does
+ * not expire.
+ *
+ * If <b>exit_source_out</b> is non-null, we set it as follows.  If we the
+ * address starts out as a non-exit address, and we remap it to an .exit
+ * address at any point, then set *<b>exit_source_out</b> to the
+ * address_entry_source_t of the first such rule.  Set *<b>exit_source_out</b>
+ * to ADDRMAPSRC_NONE if there is no such rewrite, or if the original address
+ * was a .exit.
+ */
+int
+addressmap_rewrite(char *address, size_t maxlen,
+                   unsigned flags,
+                   time_t *expires_out,
+                   addressmap_entry_source_t *exit_source_out)
+{
+  addressmap_entry_t *ent;
+  int rewrites;
+  time_t expires = TIME_MAX;
+  addressmap_entry_source_t exit_source = ADDRMAPSRC_NONE;
+  char *addr_orig = tor_strdup(address);
+  char *log_addr_orig = NULL;
+
+  /* We use a loop here to limit the total number of rewrites we do,
+   * so that we can't hit an infinite loop. */
+  for (rewrites = 0; rewrites < 16; rewrites++) {
+    int exact_match = 0;
+    log_addr_orig = tor_strdup(escaped_safe_str_client(address));
+
+    /* First check to see if there's an exact match for this address */
+    ent = strmap_get(addressmap, address);
+
+    if (!ent || !ent->new_address) {
+      /* And if we don't have an exact match, try to check whether
+       * we have a pattern-based match.
+       */
+      ent = addressmap_match_superdomains(address);
+    } else {
+      if (ent->src_wildcard && !ent->dst_wildcard &&
+          !strcasecmp(address, ent->new_address)) {
+        /* This is a rule like "rewrite *.example.com to example.com", and we
+         * just got "example.com". Instead of calling it an infinite loop,
+         * call it complete. */
+        goto done;
+      }
+      exact_match = 1;
+    }
+
+    if (!ent || !ent->new_address) {
+      /* We still have no match at all.  We're done! */
+      goto done;
+    }
+
+    /* Check wither the flags we were passed tell us not to use this
+     * mapping. */
+    switch (ent->source) {
+      case ADDRMAPSRC_DNS:
+        {
+          sa_family_t f;
+          tor_addr_t tmp;
+          f = tor_addr_parse(&tmp, ent->new_address);
+          if (f == AF_INET && !(flags & AMR_FLAG_USE_IPV4_DNS))
+            goto done;
+          else if (f == AF_INET6 && !(flags & AMR_FLAG_USE_IPV6_DNS))
+            goto done;
+        }
+        break;
+      case ADDRMAPSRC_CONTROLLER:
+      case ADDRMAPSRC_TORRC:
+        if (!(flags & AMR_FLAG_USE_MAPADDRESS))
+          goto done;
+        break;
+      case ADDRMAPSRC_AUTOMAP:
+        if (!(flags & AMR_FLAG_USE_AUTOMAP))
+          goto done;
+        break;
+      case ADDRMAPSRC_TRACKEXIT:
+        if (!(flags & AMR_FLAG_USE_TRACKEXIT))
+          goto done;
+        break;
+      case ADDRMAPSRC_NONE:
+      default:
+        log_warn(LD_BUG, "Unknown addrmap source value %d. Ignoring it.",
+                 (int) ent->source);
+        goto done;
+    }
+
+    /* Now fill in the address with the new address. That might be via
+     * appending some new stuff to the end, or via just replacing it. */
+    if (ent->dst_wildcard && !exact_match) {
+      strlcat(address, ".", maxlen);
+      strlcat(address, ent->new_address, maxlen);
+    } else {
+      strlcpy(address, ent->new_address, maxlen);
+    }
+
+    /* Is this now a .exit address?  If so, remember where we got it.*/
+    if (!strcmpend(address, ".exit") &&
+        strcmpend(addr_orig, ".exit") &&
+        exit_source == ADDRMAPSRC_NONE) {
+      exit_source = ent->source;
+    }
+
+    log_info(LD_APP, "Addressmap: rewriting %s to %s",
+             log_addr_orig, escaped_safe_str_client(address));
+    if (ent->expires > 1 && ent->expires < expires)
+      expires = ent->expires;
+
+    tor_free(log_addr_orig);
+  }
+  log_warn(LD_CONFIG,
+           "Loop detected: we've rewritten %s 16 times! Using it as-is.",
+           escaped_safe_str_client(address));
+  /* it's fine to rewrite a rewrite, but don't loop forever */
+
+ done:
+  tor_free(addr_orig);
+  tor_free(log_addr_orig);
+  if (exit_source_out)
+    *exit_source_out = exit_source;
+  if (expires_out)
+    *expires_out = expires;
+  return (rewrites > 0);
+}
+
+/** If we have a cached reverse DNS entry for the address stored in the
+ * <b>maxlen</b>-byte buffer <b>address</b> (typically, a dotted quad) then
+ * rewrite to the cached value and return 1.  Otherwise return 0.  Set
+ * *<b>expires_out</b> to the expiry time of the result, or to <b>time_max</b>
+ * if the result does not expire. */
+int
+addressmap_rewrite_reverse(char *address, size_t maxlen, unsigned flags,
+                           time_t *expires_out)
+{
+  char *s, *cp;
+  addressmap_entry_t *ent;
+  int r = 0;
+  {
+    sa_family_t f;
+    tor_addr_t tmp;
+    f = tor_addr_parse(&tmp, address);
+    if (f == AF_INET && !(flags & AMR_FLAG_USE_IPV4_DNS))
+      return 0;
+    else if (f == AF_INET6 && !(flags & AMR_FLAG_USE_IPV6_DNS))
+      return 0;
+    /* FFFF we should reverse-map virtual addresses even if we haven't
+     * enabled DNS cacheing. */
+  }
+
+  tor_asprintf(&s, "REVERSE[%s]", address);
+  ent = strmap_get(addressmap, s);
+  if (ent) {
+    cp = tor_strdup(escaped_safe_str_client(ent->new_address));
+    log_info(LD_APP, "Rewrote reverse lookup %s -> %s",
+             escaped_safe_str_client(s), cp);
+    tor_free(cp);
+    strlcpy(address, ent->new_address, maxlen);
+    r = 1;
+  }
+
+  if (expires_out)
+    *expires_out = (ent && ent->expires > 1) ? ent->expires : TIME_MAX;
+
+  tor_free(s);
+  return r;
+}
+
+/** Return 1 if <b>address</b> is already registered, else return 0. If address
+ * is already registered, and <b>update_expires</b> is non-zero, then update
+ * the expiry time on the mapping with update_expires if it is a
+ * mapping created by TrackHostExits. */
+int
+addressmap_have_mapping(const char *address, int update_expiry)
+{
+  addressmap_entry_t *ent;
+  if (!(ent=strmap_get_lc(addressmap, address)))
+    return 0;
+  if (update_expiry && ent->source==ADDRMAPSRC_TRACKEXIT)
+    ent->expires=time(NULL) + update_expiry;
+  return 1;
+}
+
+/** Register a request to map <b>address</b> to <b>new_address</b>,
+ * which will expire on <b>expires</b> (or 0 if never expires from
+ * config file, 1 if never expires from controller, 2 if never expires
+ * (virtual address mapping) from the controller.)
+ *
+ * <b>new_address</b> should be a newly dup'ed string, which we'll use or
+ * free as appropriate. We will leave <b>address</b> alone.
+ *
+ * If <b>wildcard_addr</b> is true, then the mapping will match any address
+ * equal to <b>address</b>, or any address ending with a period followed by
+ * <b>address</b>.  If <b>wildcard_addr</b> and <b>wildcard_new_addr</b> are
+ * both true, the mapping will rewrite addresses that end with
+ * ".<b>address</b>" into ones that end with ".<b>new_address</b>".
+ *
+ * If <b>new_address</b> is NULL, or <b>new_address</b> is equal to
+ * <b>address</b> and <b>wildcard_addr</b> is equal to
+ * <b>wildcard_new_addr</b>, remove any mappings that exist from
+ * <b>address</b>.
+ *
+ * It is an error to set <b>wildcard_new_addr</b> if <b>wildcard_addr</b> is
+ * not set. */
+void
+addressmap_register(const char *address, char *new_address, time_t expires,
+                    addressmap_entry_source_t source,
+                    const int wildcard_addr,
+                    const int wildcard_new_addr)
+{
+  addressmap_entry_t *ent;
+
+  if (wildcard_new_addr)
+    tor_assert(wildcard_addr);
+
+  ent = strmap_get(addressmap, address);
+  if (!new_address || (!strcasecmp(address,new_address) &&
+                       wildcard_addr == wildcard_new_addr)) {
+    /* Remove the mapping, if any. */
+    tor_free(new_address);
+    if (ent) {
+      addressmap_ent_remove(address,ent);
+      strmap_remove(addressmap, address);
+    }
+    return;
+  }
+  if (!ent) { /* make a new one and register it */
+    ent = tor_malloc_zero(sizeof(addressmap_entry_t));
+    strmap_set(addressmap, address, ent);
+  } else if (ent->new_address) { /* we need to clean up the old mapping. */
+    if (expires > 1) {
+      log_info(LD_APP,"Temporary addressmap ('%s' to '%s') not performed, "
+               "since it's already mapped to '%s'",
+               safe_str_client(address),
+               safe_str_client(new_address),
+               safe_str_client(ent->new_address));
+      tor_free(new_address);
+      return;
+    }
+    if (address_is_in_virtual_range(ent->new_address) &&
+        expires != 2) {
+      /* XXX This isn't the perfect test; we want to avoid removing
+       * mappings set from the control interface _as virtual mapping */
+      addressmap_virtaddress_remove(address, ent);
+    }
+    tor_free(ent->new_address);
+  } /* else { we have an in-progress resolve with no mapping. } */
+
+  ent->new_address = new_address;
+  ent->expires = expires==2 ? 1 : expires;
+  ent->num_resolve_failures = 0;
+  ent->source = source;
+  ent->src_wildcard = wildcard_addr ? 1 : 0;
+  ent->dst_wildcard = wildcard_new_addr ? 1 : 0;
+
+  log_info(LD_CONFIG, "Addressmap: (re)mapped '%s' to '%s'",
+           safe_str_client(address),
+           safe_str_client(ent->new_address));
+  control_event_address_mapped(address, ent->new_address, expires, NULL, 1);
+}
+
+/** An attempt to resolve <b>address</b> failed at some OR.
+ * Increment the number of resolve failures we have on record
+ * for it, and then return that number.
+ */
+int
+client_dns_incr_failures(const char *address)
+{
+  addressmap_entry_t *ent = strmap_get(addressmap, address);
+  if (!ent) {
+    ent = tor_malloc_zero(sizeof(addressmap_entry_t));
+    ent->expires = time(NULL) + MAX_DNS_ENTRY_AGE;
+    strmap_set(addressmap,address,ent);
+  }
+  if (ent->num_resolve_failures < SHRT_MAX)
+    ++ent->num_resolve_failures; /* don't overflow */
+  log_info(LD_APP, "Address %s now has %d resolve failures.",
+           safe_str_client(address),
+           ent->num_resolve_failures);
+  return ent->num_resolve_failures;
+}
+
+/** If <b>address</b> is in the client DNS addressmap, reset
+ * the number of resolve failures we have on record for it.
+ * This is used when we fail a stream because it won't resolve:
+ * otherwise future attempts on that address will only try once.
+ */
+void
+client_dns_clear_failures(const char *address)
+{
+  addressmap_entry_t *ent = strmap_get(addressmap, address);
+  if (ent)
+    ent->num_resolve_failures = 0;
+}
+
+/** Record the fact that <b>address</b> resolved to <b>name</b>.
+ * We can now use this in subsequent streams via addressmap_rewrite()
+ * so we can more correctly choose an exit that will allow <b>address</b>.
+ *
+ * If <b>exitname</b> is defined, then append the addresses with
+ * ".exitname.exit" before registering the mapping.
+ *
+ * If <b>ttl</b> is nonnegative, the mapping will be valid for
+ * <b>ttl</b>seconds; otherwise, we use the default.
+ */
+static void
+client_dns_set_addressmap_impl(entry_connection_t *for_conn,
+                               const char *address, const char *name,
+                               const char *exitname,
+                               int ttl)
+{
+  char *extendedaddress=NULL, *extendedval=NULL;
+  (void)for_conn;
+
+  tor_assert(address);
+  tor_assert(name);
+
+  if (ttl<0)
+    ttl = DEFAULT_DNS_TTL;
+  else
+    ttl = dns_clip_ttl(ttl);
+
+  if (exitname) {
+    /* XXXX fails to ever get attempts to get an exit address of
+     * google.com.digest[=~]nickname.exit; we need a syntax for this that
+     * won't make strict RFC952-compliant applications (like us) barf. */
+    tor_asprintf(&extendedaddress,
+                 "%s.%s.exit", address, exitname);
+    tor_asprintf(&extendedval,
+                 "%s.%s.exit", name, exitname);
+  } else {
+    tor_asprintf(&extendedaddress,
+                 "%s", address);
+    tor_asprintf(&extendedval,
+                 "%s", name);
+  }
+  addressmap_register(extendedaddress, extendedval,
+                      time(NULL) + ttl, ADDRMAPSRC_DNS, 0, 0);
+  tor_free(extendedaddress);
+}
+
+/** Record the fact that <b>address</b> resolved to <b>val</b>.
+ * We can now use this in subsequent streams via addressmap_rewrite()
+ * so we can more correctly choose an exit that will allow <b>address</b>.
+ *
+ * If <b>exitname</b> is defined, then append the addresses with
+ * ".exitname.exit" before registering the mapping.
+ *
+ * If <b>ttl</b> is nonnegative, the mapping will be valid for
+ * <b>ttl</b>seconds; otherwise, we use the default.
+ */
+void
+client_dns_set_addressmap(entry_connection_t *for_conn,
+                          const char *address,
+                          const tor_addr_t *val,
+                          const char *exitname,
+                          int ttl)
+{
+  tor_addr_t addr_tmp;
+  char valbuf[TOR_ADDR_BUF_LEN];
+
+  tor_assert(address);
+  tor_assert(val);
+
+  if (tor_addr_parse(&addr_tmp, address) >= 0)
+    return; /* If address was an IP address already, don't add a mapping. */
+
+  if (tor_addr_family(val) == AF_INET) {
+    if (! for_conn->entry_cfg.cache_ipv4_answers)
+      return;
+  } else if (tor_addr_family(val) == AF_INET6) {
+    if (! for_conn->entry_cfg.cache_ipv6_answers)
+      return;
+  }
+
+  if (! tor_addr_to_str(valbuf, val, sizeof(valbuf), 1))
+    return;
+
+  client_dns_set_addressmap_impl(for_conn, address, valbuf, exitname, ttl);
+}
+
+/** Add a cache entry noting that <b>address</b> (ordinarily a dotted quad)
+ * resolved via a RESOLVE_PTR request to the hostname <b>v</b>.
+ *
+ * If <b>exitname</b> is defined, then append the addresses with
+ * ".exitname.exit" before registering the mapping.
+ *
+ * If <b>ttl</b> is nonnegative, the mapping will be valid for
+ * <b>ttl</b>seconds; otherwise, we use the default.
+ */
+void
+client_dns_set_reverse_addressmap(entry_connection_t *for_conn,
+                                  const char *address, const char *v,
+                                  const char *exitname,
+                                  int ttl)
+{
+  char *s = NULL;
+  {
+    tor_addr_t tmp_addr;
+    sa_family_t f = tor_addr_parse(&tmp_addr, address);
+    if ((f == AF_INET && ! for_conn->entry_cfg.cache_ipv4_answers) ||
+        (f == AF_INET6 && ! for_conn->entry_cfg.cache_ipv6_answers))
+      return;
+  }
+  tor_asprintf(&s, "REVERSE[%s]", address);
+  client_dns_set_addressmap_impl(for_conn, s, v, exitname, ttl);
+  tor_free(s);
+}
+
+/* By default, we hand out 127.192.0.1 through 127.254.254.254.
+ * These addresses should map to localhost, so even if the
+ * application accidentally tried to connect to them directly (not
+ * via Tor), it wouldn't get too far astray.
+ *
+ * These options are configured by parse_virtual_addr_network().
+ */
+
+static virtual_addr_conf_t virtaddr_conf_ipv4;
+static virtual_addr_conf_t virtaddr_conf_ipv6;
+
+/** Read a netmask of the form 127.192.0.0/10 from "val", and check whether
+ * it's a valid set of virtual addresses to hand out in response to MAPADDRESS
+ * requests.  Return 0 on success; set *msg (if provided) to a newly allocated
+ * string and return -1 on failure.  If validate_only is false, sets the
+ * actual virtual address range to the parsed value. */
+int
+parse_virtual_addr_network(const char *val, sa_family_t family,
+                           int validate_only,
+                           char **msg)
+{
+  const int ipv6 = (family == AF_INET6);
+  tor_addr_t addr;
+  maskbits_t bits;
+  const int max_prefix_bits = ipv6 ? 104 : 16;
+  virtual_addr_conf_t *conf = ipv6 ? &virtaddr_conf_ipv6 : &virtaddr_conf_ipv4;
+
+  if (!val || val[0] == '\0') {
+    if (msg)
+      tor_asprintf(msg, "Value not present (%s) after VirtualAddressNetwork%s",
+                   val?"Empty":"NULL", ipv6?"IPv6":"");
+    return -1;
+  }
+  if (tor_addr_parse_mask_ports(val, 0, &addr, &bits, NULL, NULL) < 0) {
+    if (msg)
+      tor_asprintf(msg, "Error parsing VirtualAddressNetwork%s %s",
+                   ipv6?"IPv6":"", val);
+    return -1;
+  }
+  if (tor_addr_family(&addr) != family) {
+    if (msg)
+      tor_asprintf(msg, "Incorrect address type for VirtualAddressNetwork%s",
+                   ipv6?"IPv6":"");
+    return -1;
+  }
+#if 0
+  if (port_min != 1 || port_max != 65535) {
+    if (msg)
+      tor_asprintf(msg, "Can't specify ports on VirtualAddressNetwork%s",
+                   ipv6?"IPv6":"");
+    return -1;
+  }
+#endif /* 0 */
+
+  if (bits > max_prefix_bits) {
+    if (msg)
+      tor_asprintf(msg, "VirtualAddressNetwork%s expects a /%d "
+                   "network or larger",ipv6?"IPv6":"", max_prefix_bits);
+    return -1;
+  }
+
+  if (validate_only)
+    return 0;
+
+  tor_addr_copy(&conf->addr, &addr);
+  conf->bits = bits;
+
+  return 0;
+}
+
+/**
+ * Return true iff <b>addr</b> is likely to have been returned by
+ * client_dns_get_unused_address.
+ **/
+int
+address_is_in_virtual_range(const char *address)
+{
+  tor_addr_t addr;
+  tor_assert(address);
+  if (!strcasecmpend(address, ".virtual")) {
+    return 1;
+  } else if (tor_addr_parse(&addr, address) >= 0) {
+    const virtual_addr_conf_t *conf = (tor_addr_family(&addr) == AF_INET6) ?
+      &virtaddr_conf_ipv6 : &virtaddr_conf_ipv4;
+    if (tor_addr_compare_masked(&addr, &conf->addr, conf->bits, CMP_EXACT)==0)
+      return 1;
+  }
+  return 0;
+}
+
+/** Return a random address conforming to the virtual address configuration
+ * in <b>conf</b>.
+ */
+STATIC void
+get_random_virtual_addr(const virtual_addr_conf_t *conf, tor_addr_t *addr_out)
+{
+  uint8_t tmp[4];
+  const uint8_t *addr_bytes;
+  uint8_t bytes[16];
+  const int ipv6 = tor_addr_family(&conf->addr) == AF_INET6;
+  const int total_bytes = ipv6 ? 16 : 4;
+
+  tor_assert(conf->bits <= total_bytes * 8);
+
+  /* Set addr_bytes to the bytes of the virtual network, in host order */
+  if (ipv6) {
+    addr_bytes = tor_addr_to_in6_addr8(&conf->addr);
+  } else {
+    set_uint32(tmp, tor_addr_to_ipv4n(&conf->addr));
+    addr_bytes = tmp;
+  }
+
+  /* Get an appropriate number of random bytes. */
+  crypto_rand((char*)bytes, total_bytes);
+
+  /* Now replace the first "conf->bits" bits of 'bytes' with addr_bytes*/
+  if (conf->bits >= 8)
+    memcpy(bytes, addr_bytes, conf->bits / 8);
+  if (conf->bits & 7) {
+    uint8_t mask = 0xff >> (conf->bits & 7);
+    bytes[conf->bits/8] &= mask;
+    bytes[conf->bits/8] |= addr_bytes[conf->bits/8] & ~mask;
+  }
+
+  if (ipv6)
+    tor_addr_from_ipv6_bytes(addr_out, (char*) bytes);
+  else
+    tor_addr_from_ipv4n(addr_out, get_uint32(bytes));
+
+  tor_assert(tor_addr_compare_masked(addr_out, &conf->addr,
+                                     conf->bits, CMP_EXACT)==0);
+}
+
+/** Return a newly allocated string holding an address of <b>type</b>
+ * (one of RESOLVED_TYPE_{IPV4|IPV6|HOSTNAME}) that has not yet been
+ * mapped, and that is very unlikely to be the address of any real host.
+ *
+ * May return NULL if we have run out of virtual addresses.
+ */
+static char *
+addressmap_get_virtual_address(int type)
+{
+  char buf[64];
+  tor_assert(addressmap);
+
+  if (type == RESOLVED_TYPE_HOSTNAME) {
+    char rand_bytes[10];
+    do {
+      crypto_rand(rand_bytes, sizeof(rand_bytes));
+      base32_encode(buf,sizeof(buf),rand_bytes,sizeof(rand_bytes));
+      strlcat(buf, ".virtual", sizeof(buf));
+    } while (strmap_get(addressmap, buf));
+    return tor_strdup(buf);
+  } else if (type == RESOLVED_TYPE_IPV4 || type == RESOLVED_TYPE_IPV6) {
+    const int ipv6 = (type == RESOLVED_TYPE_IPV6);
+    const virtual_addr_conf_t *conf = ipv6 ?
+      &virtaddr_conf_ipv6 : &virtaddr_conf_ipv4;
+
+    /* Don't try more than 1000 times.  This gives us P < 1e-9 for
+     * failing to get a good address so long as the address space is
+     * less than ~97.95% full.  That's always going to be true under
+     * sensible circumstances for an IPv6 /10, and it's going to be
+     * true for an IPv4 /10 as long as we've handed out less than
+     * 4.08 million addresses. */
+    uint32_t attempts = 1000;
+
+    tor_addr_t addr;
+
+    while (attempts--) {
+      get_random_virtual_addr(conf, &addr);
+
+      if (!ipv6) {
+        /* Don't hand out any .0 or .255 address. */
+        const uint32_t a = tor_addr_to_ipv4h(&addr);
+        if ((a & 0xff) == 0 || (a & 0xff) == 0xff)
+          continue;
+      }
+
+      tor_addr_to_str(buf, &addr, sizeof(buf), 1);
+      if (!strmap_get(addressmap, buf)) {
+        /* XXXX This code is to make sure I didn't add an undecorated version
+         * by mistake. I hope it's needless. */
+        char tmp[TOR_ADDR_BUF_LEN];
+        tor_addr_to_str(tmp, &addr, sizeof(tmp), 0);
+        if (strmap_get(addressmap, tmp)) {
+          // LCOV_EXCL_START
+          log_warn(LD_BUG, "%s wasn't in the addressmap, but %s was.",
+                   buf, tmp);
+          continue;
+          // LCOV_EXCL_STOP
+        }
+
+        return tor_strdup(buf);
+      }
+    }
+    log_warn(LD_CONFIG, "Ran out of virtual addresses!");
+    return NULL;
+  } else {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Called with unsupported address type (%d)", type);
+    return NULL;
+    // LCOV_EXCL_STOP
+  }
+}
+
+/** A controller has requested that we map some address of type
+ * <b>type</b> to the address <b>new_address</b>.  Choose an address
+ * that is unlikely to be used, and map it, and return it in a newly
+ * allocated string.  If another address of the same type is already
+ * mapped to <b>new_address</b>, try to return a copy of that address.
+ *
+ * The string in <b>new_address</b> may be freed or inserted into a map
+ * as appropriate.  May return NULL if are out of virtual addresses.
+ **/
+const char *
+addressmap_register_virtual_address(int type, char *new_address)
+{
+  char **addrp;
+  virtaddress_entry_t *vent;
+  int vent_needs_to_be_added = 0;
+
+  tor_assert(new_address);
+  tor_assert(addressmap);
+  tor_assert(virtaddress_reversemap);
+
+  vent = strmap_get(virtaddress_reversemap, new_address);
+  if (!vent) {
+    vent = tor_malloc_zero(sizeof(virtaddress_entry_t));
+    vent_needs_to_be_added = 1;
+  }
+
+  if (type == RESOLVED_TYPE_IPV4)
+    addrp = &vent->ipv4_address;
+  else if (type == RESOLVED_TYPE_IPV6)
+    addrp = &vent->ipv6_address;
+  else
+    addrp = &vent->hostname_address;
+
+  if (*addrp) {
+    addressmap_entry_t *ent = strmap_get(addressmap, *addrp);
+    if (ent && ent->new_address &&
+        !strcasecmp(new_address, ent->new_address)) {
+      tor_free(new_address);
+      tor_assert(!vent_needs_to_be_added);
+      return *addrp;
+    } else {
+      log_warn(LD_BUG,
+               "Internal confusion: I thought that '%s' was mapped to by "
+               "'%s', but '%s' really maps to '%s'. This is a harmless bug.",
+               safe_str_client(new_address),
+               safe_str_client(*addrp),
+               safe_str_client(*addrp),
+               ent?safe_str_client(ent->new_address):"(nothing)");
+    }
+  }
+
+  tor_free(*addrp);
+  *addrp = addressmap_get_virtual_address(type);
+  if (!*addrp) {
+    tor_free(vent);
+    tor_free(new_address);
+    return NULL;
+  }
+  log_info(LD_APP, "Registering map from %s to %s", *addrp, new_address);
+  if (vent_needs_to_be_added)
+    strmap_set(virtaddress_reversemap, new_address, vent);
+  addressmap_register(*addrp, new_address, 2, ADDRMAPSRC_AUTOMAP, 0, 0);
+
+  /* FFFF register corresponding reverse mapping. */
+
+#if 0
+  {
+    /* Try to catch possible bugs */
+    addressmap_entry_t *ent;
+    ent = strmap_get(addressmap, *addrp);
+    tor_assert(ent);
+    tor_assert(!strcasecmp(ent->new_address,new_address));
+    vent = strmap_get(virtaddress_reversemap, new_address);
+    tor_assert(vent);
+    tor_assert(!strcasecmp(*addrp,
+                           (type == RESOLVED_TYPE_IPV4) ?
+                           vent->ipv4_address : vent->hostname_address));
+    log_info(LD_APP, "Map from %s to %s okay.",
+             safe_str_client(*addrp),
+             safe_str_client(new_address));
+  }
+#endif /* 0 */
+
+  return *addrp;
+}
+
+/** Return 1 if <b>address</b> has funny characters in it like colons. Return
+ * 0 if it's fine, or if we're configured to allow it anyway.  <b>client</b>
+ * should be true if we're using this address as a client; false if we're
+ * using it as a server.
+ */
+int
+address_is_invalid_destination(const char *address, int client)
+{
+  if (client) {
+    if (get_options()->AllowNonRFC953Hostnames)
+      return 0;
+  } else {
+    if (get_options()->ServerDNSAllowNonRFC953Hostnames)
+      return 0;
+  }
+
+  /* It might be an IPv6 address! */
+  {
+    tor_addr_t a;
+    if (tor_addr_parse(&a, address) >= 0)
+      return 0;
+  }
+
+  while (*address) {
+    if (TOR_ISALNUM(*address) ||
+        *address == '-' ||
+        *address == '.' ||
+        *address == '_') /* Underscore is not allowed, but Windows does it
+                          * sometimes, just to thumb its nose at the IETF. */
+      ++address;
+    else
+      return 1;
+  }
+  return 0;
+}
+
+/** Iterate over all address mappings which have expiry times between
+ * min_expires and max_expires, inclusive.  If sl is provided, add an
+ * "old-addr new-addr expiry" string to sl for each mapping, omitting
+ * the expiry time if want_expiry is false. If sl is NULL, remove the
+ * mappings.
+ */
+void
+addressmap_get_mappings(smartlist_t *sl, time_t min_expires,
+                        time_t max_expires, int want_expiry)
+{
+   strmap_iter_t *iter;
+   const char *key;
+   void *val_;
+   addressmap_entry_t *val;
+
+   if (!addressmap)
+     addressmap_init();
+
+   for (iter = strmap_iter_init(addressmap); !strmap_iter_done(iter); ) {
+     strmap_iter_get(iter, &key, &val_);
+     val = val_;
+     if (val->expires >= min_expires && val->expires <= max_expires) {
+       if (!sl) {
+         iter = strmap_iter_next_rmv(addressmap,iter);
+         addressmap_ent_remove(key, val);
+         continue;
+       } else if (val->new_address) {
+         const char *src_wc = val->src_wildcard ? "*." : "";
+         const char *dst_wc = val->dst_wildcard ? "*." : "";
+         if (want_expiry) {
+           if (val->expires < 3 || val->expires == TIME_MAX)
+             smartlist_add_asprintf(sl, "%s%s %s%s NEVER",
+                                    src_wc, key, dst_wc, val->new_address);
+           else {
+             char isotime[ISO_TIME_LEN+1];
+             format_iso_time(isotime, val->expires);
+             smartlist_add_asprintf(sl, "%s%s %s%s \"%s\"",
+                                    src_wc, key, dst_wc, val->new_address,
+                                    isotime);
+           }
+         } else {
+           smartlist_add_asprintf(sl, "%s%s %s%s",
+                                  src_wc, key, dst_wc, val->new_address);
+         }
+       }
+     }
+     iter = strmap_iter_next(addressmap,iter);
+   }
+}
diff --git a/src/feature/client/addressmap.h b/src/feature/client/addressmap.h
new file mode 100644
index 0000000000..b0db5c8b4e
--- /dev/null
+++ b/src/feature/client/addressmap.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_ADDRESSMAP_H
+#define TOR_ADDRESSMAP_H
+
+#include "lib/testsupport/testsupport.h"
+
+void addressmap_init(void);
+void addressmap_clear_excluded_trackexithosts(const or_options_t *options);
+void addressmap_clear_invalid_automaps(const or_options_t *options);
+void addressmap_clean(time_t now);
+void addressmap_clear_configured(void);
+void addressmap_clear_transient(void);
+void addressmap_free_all(void);
+#define AMR_FLAG_USE_IPV4_DNS   (1u<<0)
+#define AMR_FLAG_USE_IPV6_DNS   (1u<<1)
+#define AMR_FLAG_USE_MAPADDRESS (1u<<2)
+#define AMR_FLAG_USE_AUTOMAP    (1u<<3)
+#define AMR_FLAG_USE_TRACKEXIT  (1u<<4)
+int addressmap_rewrite(char *address, size_t maxlen, unsigned flags,
+                       time_t *expires_out,
+                       addressmap_entry_source_t *exit_source_out);
+int addressmap_rewrite_reverse(char *address, size_t maxlen, unsigned flags,
+                               time_t *expires_out);
+int addressmap_have_mapping(const char *address, int update_timeout);
+
+void addressmap_register(const char *address, char *new_address,
+                         time_t expires, addressmap_entry_source_t source,
+                         const int address_wildcard,
+                         const int new_address_wildcard);
+int parse_virtual_addr_network(const char *val,
+                               sa_family_t family, int validate_only,
+                               char **msg);
+int client_dns_incr_failures(const char *address);
+void client_dns_clear_failures(const char *address);
+void client_dns_set_addressmap(entry_connection_t *for_conn,
+                               const char *address, const tor_addr_t *val,
+                               const char *exitname, int ttl);
+const char *addressmap_register_virtual_address(int type, char *new_address);
+void addressmap_get_mappings(smartlist_t *sl, time_t min_expires,
+                             time_t max_expires, int want_expiry);
+int address_is_in_virtual_range(const char *addr);
+void clear_trackexithost_mappings(const char *exitname);
+void client_dns_set_reverse_addressmap(entry_connection_t *for_conn,
+                                       const char *address, const char *v,
+                                       const char *exitname, int ttl);
+int addressmap_address_should_automap(const char *address,
+                                      const or_options_t *options);
+
+#ifdef ADDRESSMAP_PRIVATE
+typedef struct virtual_addr_conf_t {
+  tor_addr_t addr;
+  maskbits_t bits;
+} virtual_addr_conf_t;
+
+STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf,
+                                    tor_addr_t *addr_out);
+#endif /* defined(ADDRESSMAP_PRIVATE) */
+
+#endif /* !defined(TOR_ADDRESSMAP_H) */
+
diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c
new file mode 100644
index 0000000000..ca0a13f2a0
--- /dev/null
+++ b/src/feature/client/bridges.c
@@ -0,0 +1,937 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bridges.c
+ * \brief Code to manage bridges and bridge selection.
+ *
+ * Bridges are fixed entry nodes, used for censorship circumvention.
+ **/
+
+#define TOR_BRIDGES_PRIVATE
+
+#include "or/or.h"
+#include "or/bridges.h"
+#include "or/circuitbuild.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/directory.h"
+#include "or/entrynodes.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerset.h"
+#include "or/transports.h"
+
+#include "or/extend_info_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerstatus_st.h"
+
+/** Information about a configured bridge. Currently this just matches the
+ * ones in the torrc file, but one day we may be able to learn about new
+ * bridges on our own, and remember them in the state file. */
+struct bridge_info_t {
+  /** Address and port of the bridge, as configured by the user.*/
+  tor_addr_port_t addrport_configured;
+  /** Address of the bridge. */
+  tor_addr_t addr;
+  /** TLS port for the bridge. */
+  uint16_t port;
+  /** Boolean: We are re-parsing our bridge list, and we are going to remove
+   * this one if we don't find it in the list of configured bridges. */
+  unsigned marked_for_removal : 1;
+  /** Expected identity digest, or all zero bytes if we don't know what the
+   * digest should be. */
+  char identity[DIGEST_LEN];
+
+  /** Name of pluggable transport protocol taken from its config line. */
+  char *transport_name;
+
+  /** When should we next try to fetch a descriptor for this bridge? */
+  download_status_t fetch_status;
+
+  /** A smartlist of k=v values to be passed to the SOCKS proxy, if
+      transports are used for this bridge. */
+  smartlist_t *socks_args;
+};
+
+#define bridge_free(bridge) \
+  FREE_AND_NULL(bridge_info_t, bridge_free_, (bridge))
+
+static void bridge_free_(bridge_info_t *bridge);
+static void rewrite_node_address_for_bridge(const bridge_info_t *bridge,
+                                            node_t *node);
+
+/** A list of configured bridges. Whenever we actually get a descriptor
+ * for one, we add it as an entry guard.  Note that the order of bridges
+ * in this list does not necessarily correspond to the order of bridges
+ * in the torrc. */
+static smartlist_t *bridge_list = NULL;
+
+/** Mark every entry of the bridge list to be removed on our next call to
+ * sweep_bridge_list unless it has first been un-marked. */
+void
+mark_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b,
+                    b->marked_for_removal = 1);
+}
+
+/** Remove every entry of the bridge list that was marked with
+ * mark_bridge_list if it has not subsequently been un-marked. */
+void
+sweep_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+    if (b->marked_for_removal) {
+      SMARTLIST_DEL_CURRENT(bridge_list, b);
+      bridge_free(b);
+    }
+  } SMARTLIST_FOREACH_END(b);
+}
+
+/** Initialize the bridge list to empty, creating it if needed. */
+STATIC void
+clear_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b, bridge_free(b));
+  smartlist_clear(bridge_list);
+}
+
+/** Free the bridge <b>bridge</b>. */
+static void
+bridge_free_(bridge_info_t *bridge)
+{
+  if (!bridge)
+    return;
+
+  tor_free(bridge->transport_name);
+  if (bridge->socks_args) {
+    SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s));
+    smartlist_free(bridge->socks_args);
+  }
+
+  tor_free(bridge);
+}
+
+/** Return a list of all the configured bridges, as bridge_info_t pointers. */
+const smartlist_t *
+bridge_list_get(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  return bridge_list;
+}
+
+/**
+ * Given a <b>bridge</b>, return a pointer to its RSA identity digest, or
+ * NULL if we don't know one for it.
+ */
+const uint8_t *
+bridge_get_rsa_id_digest(const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  if (tor_digest_is_zero(bridge->identity))
+    return NULL;
+  else
+    return (const uint8_t *) bridge->identity;
+}
+
+/**
+ * Given a <b>bridge</b>, return a pointer to its configured addr:port
+ * combination.
+ */
+const tor_addr_port_t *
+bridge_get_addr_port(const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  return &bridge->addrport_configured;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches any of the
+ * tor_addr_port_t's in <b>orports</b>, return that bridge.  Else return
+ * NULL. */
+STATIC bridge_info_t *
+get_configured_bridge_by_orports_digest(const char *digest,
+                                        const smartlist_t *orports)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      if (tor_digest_is_zero(bridge->identity)) {
+        SMARTLIST_FOREACH_BEGIN(orports, tor_addr_port_t *, ap)
+          {
+            if (tor_addr_compare(&bridge->addr, &ap->addr, CMP_EXACT) == 0 &&
+                bridge->port == ap->port)
+              return bridge;
+          }
+        SMARTLIST_FOREACH_END(ap);
+      }
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    }
+  SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
+ * return that bridge.  Else return NULL. If <b>digest</b> is NULL, check for
+ * address/port matches only. */
+bridge_info_t *
+get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      if ((tor_digest_is_zero(bridge->identity) || digest == NULL) &&
+          !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
+          bridge->port == port)
+        return bridge;
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    }
+  SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/**
+ * As get_configured_bridge_by_addr_port, but require that the
+ * address match <b>addr</b>:<b>port</b>, and that the ID digest match
+ * <b>digest</b>.  (The other function will ignore the address if the
+ * digest matches.)
+ */
+bridge_info_t *
+get_configured_bridge_by_exact_addr_port_digest(const tor_addr_t *addr,
+                                                uint16_t port,
+                                                const char *digest)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
+    if (!tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
+        bridge->port == port) {
+
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+      else if (!digest || tor_digest_is_zero(bridge->identity))
+        return bridge;
+    }
+
+  } SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
+ * return 1.  Else return 0. If <b>digest</b> is NULL, check for
+ * address/port matches only. */
+int
+addr_is_a_configured_bridge(const tor_addr_t *addr,
+                                uint16_t port,
+                                const char *digest)
+{
+  tor_assert(addr);
+  return get_configured_bridge_by_addr_port_digest(addr, port, digest) ? 1 : 0;
+}
+
+/** If we have a bridge configured whose digest matches
+ * <b>ei->identity_digest</b>, or a bridge with no known digest whose address
+ * matches <b>ei->addr</b>:<b>ei->port</b>, return 1.  Else return 0.
+ * If <b>ei->onion_key</b> is NULL, check for address/port matches only. */
+int
+extend_info_is_a_configured_bridge(const extend_info_t *ei)
+{
+  const char *digest = ei->onion_key ? ei->identity_digest : NULL;
+  return addr_is_a_configured_bridge(&ei->addr, ei->port, digest);
+}
+
+/** Wrapper around get_configured_bridge_by_addr_port_digest() to look
+ * it up via router descriptor <b>ri</b>. */
+static bridge_info_t *
+get_configured_bridge_by_routerinfo(const routerinfo_t *ri)
+{
+  bridge_info_t *bi = NULL;
+  smartlist_t *orports = router_get_all_orports(ri);
+  bi = get_configured_bridge_by_orports_digest(ri->cache_info.identity_digest,
+                                               orports);
+  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
+  smartlist_free(orports);
+  return bi;
+}
+
+/** Return 1 if <b>ri</b> is one of our known bridges, else 0. */
+int
+routerinfo_is_a_configured_bridge(const routerinfo_t *ri)
+{
+  return get_configured_bridge_by_routerinfo(ri) ? 1 : 0;
+}
+
+/** Return 1 if <b>node</b> is one of our configured bridges, else 0. */
+int
+node_is_a_configured_bridge(const node_t *node)
+{
+  int retval = 0;
+  smartlist_t *orports = node_get_all_orports(node);
+  retval = get_configured_bridge_by_orports_digest(node->identity,
+                                                   orports) != NULL;
+  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
+  smartlist_free(orports);
+  return retval;
+}
+
+/** We made a connection to a router at <b>addr</b>:<b>port</b>
+ * without knowing its digest. Its digest turned out to be <b>digest</b>.
+ * If it was a bridge, and we still don't know its digest, record it.
+ */
+void
+learned_router_identity(const tor_addr_t *addr, uint16_t port,
+                        const char *digest,
+                        const ed25519_public_key_t *ed_id)
+{
+  // XXXX prop220 use ed_id here, once there is some way to specify
+  (void)ed_id;
+  int learned = 0;
+  bridge_info_t *bridge =
+    get_configured_bridge_by_exact_addr_port_digest(addr, port, digest);
+  if (bridge && tor_digest_is_zero(bridge->identity)) {
+    memcpy(bridge->identity, digest, DIGEST_LEN);
+    learned = 1;
+  }
+  /* XXXX prop220 remember bridge ed25519 identities -- add a field */
+#if 0
+  if (bridge && ed_id &&
+      ed25519_public_key_is_zero(&bridge->ed25519_identity) &&
+      !ed25519_public_key_is_zero(ed_id)) {
+    memcpy(&bridge->ed25519_identity, ed_id, sizeof(*ed_id));
+    learned = 1;
+  }
+#endif /* 0 */
+  if (learned) {
+    char *transport_info = NULL;
+    const char *transport_name =
+      find_transport_name_by_bridge_addrport(addr, port);
+    if (transport_name)
+      tor_asprintf(&transport_info, " (with transport '%s')", transport_name);
+
+    // XXXX prop220 log both fingerprints.
+    log_notice(LD_DIR, "Learned fingerprint %s for bridge %s%s.",
+               hex_str(digest, DIGEST_LEN), fmt_addrport(addr, port),
+               transport_info ? transport_info : "");
+    tor_free(transport_info);
+    entry_guard_learned_bridge_identity(&bridge->addrport_configured,
+                                        (const uint8_t *)digest);
+  }
+}
+
+/** Return true if <b>bridge</b> has the same identity digest as
+ *  <b>digest</b>. If <b>digest</b> is NULL, it matches
+ *  bridges with unspecified identity digests. */
+static int
+bridge_has_digest(const bridge_info_t *bridge, const char *digest)
+{
+  if (digest)
+    return tor_memeq(digest, bridge->identity, DIGEST_LEN);
+  else
+    return tor_digest_is_zero(bridge->identity);
+}
+
+/** We are about to add a new bridge at <b>addr</b>:<b>port</b>, with optional
+ * <b>digest</b> and <b>transport_name</b>. Mark for removal any previously
+ * existing bridge with the same address and port, and warn the user as
+ * appropriate.
+ */
+STATIC void
+bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port,
+                         const char *digest, const char *transport_name)
+{
+  /* Iterate the already-registered bridge list:
+
+     If you find a bridge with the same address and port, mark it for
+     removal. It doesn't make sense to have two active bridges with
+     the same IP:PORT. If the bridge in question has a different
+     digest or transport than <b>digest</b>/<b>transport_name</b>,
+     it's probably a misconfiguration and we should warn the user.
+  */
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
+    if (bridge->marked_for_removal)
+      continue;
+
+    if (tor_addr_eq(&bridge->addr, addr) && (bridge->port == port)) {
+
+      bridge->marked_for_removal = 1;
+
+      if (!bridge_has_digest(bridge, digest) ||
+          strcmp_opt(bridge->transport_name, transport_name)) {
+        /* warn the user */
+        char *bridge_description_new, *bridge_description_old;
+        tor_asprintf(&bridge_description_new, "%s:%s:%s",
+                     fmt_addrport(addr, port),
+                     digest ? hex_str(digest, DIGEST_LEN) : "",
+                     transport_name ? transport_name : "");
+        tor_asprintf(&bridge_description_old, "%s:%s:%s",
+                     fmt_addrport(&bridge->addr, bridge->port),
+                     tor_digest_is_zero(bridge->identity) ?
+                     "" : hex_str(bridge->identity,DIGEST_LEN),
+                     bridge->transport_name ? bridge->transport_name : "");
+
+        log_warn(LD_GENERAL,"Tried to add bridge '%s', but we found a conflict"
+                 " with the already registered bridge '%s'. We will discard"
+                 " the old bridge and keep '%s'. If this is not what you"
+                 " wanted, please change your configuration file accordingly.",
+                 bridge_description_new, bridge_description_old,
+                 bridge_description_new);
+
+        tor_free(bridge_description_new);
+        tor_free(bridge_description_old);
+      }
+    }
+  } SMARTLIST_FOREACH_END(bridge);
+}
+
+/** Return True if we have a bridge that uses a transport with name
+ *  <b>transport_name</b>. */
+MOCK_IMPL(int,
+transport_is_needed, (const char *transport_name))
+{
+  if (!bridge_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (bridge->transport_name &&
+        !strcmp(bridge->transport_name, transport_name))
+      return 1;
+  } SMARTLIST_FOREACH_END(bridge);
+
+  return 0;
+}
+
+/** Register the bridge information in <b>bridge_line</b> to the
+ *  bridge subsystem. Steals reference of <b>bridge_line</b>. */
+void
+bridge_add_from_config(bridge_line_t *bridge_line)
+{
+  bridge_info_t *b;
+
+  // XXXX prop220 add a way to specify ed25519 ID to bridge_line_t.
+
+  { /* Log the bridge we are about to register: */
+    log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)",
+              fmt_addrport(&bridge_line->addr, bridge_line->port),
+              bridge_line->transport_name ?
+              bridge_line->transport_name : "no transport",
+              tor_digest_is_zero(bridge_line->digest) ?
+              "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN));
+
+    if (bridge_line->socks_args) { /* print socks arguments */
+      int i = 0;
+
+      tor_assert(smartlist_len(bridge_line->socks_args) > 0);
+
+      log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:",
+                smartlist_len(bridge_line->socks_args));
+      SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg,
+                        log_debug(LD_CONFIG, "%d: %s", ++i, arg));
+    }
+  }
+
+  bridge_resolve_conflicts(&bridge_line->addr,
+                           bridge_line->port,
+                           bridge_line->digest,
+                           bridge_line->transport_name);
+
+  b = tor_malloc_zero(sizeof(bridge_info_t));
+  tor_addr_copy(&b->addrport_configured.addr, &bridge_line->addr);
+  b->addrport_configured.port = bridge_line->port;
+  tor_addr_copy(&b->addr, &bridge_line->addr);
+  b->port = bridge_line->port;
+  memcpy(b->identity, bridge_line->digest, DIGEST_LEN);
+  if (bridge_line->transport_name)
+    b->transport_name = bridge_line->transport_name;
+  b->fetch_status.schedule = DL_SCHED_BRIDGE;
+  b->fetch_status.increment_on = DL_SCHED_INCREMENT_ATTEMPT;
+  /* We can't reset the bridge's download status here, because UseBridges
+   * might be 0 now, and it might be changed to 1 much later. */
+  b->socks_args = bridge_line->socks_args;
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+
+  tor_free(bridge_line); /* Deallocate bridge_line now. */
+
+  smartlist_add(bridge_list, b);
+}
+
+/** If <b>digest</b> is one of our known bridges, return it. */
+STATIC bridge_info_t *
+find_bridge_by_digest(const char *digest)
+{
+  if (! bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, bridge,
+    {
+      if (tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    });
+  return NULL;
+}
+
+/** Given the <b>addr</b> and <b>port</b> of a bridge, if that bridge
+ *  supports a pluggable transport, return its name. Otherwise, return
+ *  NULL. */
+const char *
+find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+{
+  if (!bridge_list)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (tor_addr_eq(&bridge->addr, addr) &&
+        (bridge->port == port))
+      return bridge->transport_name;
+  } SMARTLIST_FOREACH_END(bridge);
+
+  return NULL;
+}
+
+/** If <b>addr</b> and <b>port</b> match the address and port of a
+ * bridge of ours that uses pluggable transports, place its transport
+ * in <b>transport</b>.
+ *
+ * Return 0 on success (found a transport, or found a bridge with no
+ * transport, or found no bridge); return -1 if we should be using a
+ * transport, but the transport could not be found.
+ */
+int
+get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
+                                  const transport_t **transport)
+{
+  *transport = NULL;
+  if (!bridge_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (tor_addr_eq(&bridge->addr, addr) &&
+        (bridge->port == port)) { /* bridge matched */
+      if (bridge->transport_name) { /* it also uses pluggable transports */
+        *transport = transport_get_by_name(bridge->transport_name);
+        if (*transport == NULL) { /* it uses pluggable transports, but
+                                     the transport could not be found! */
+          return -1;
+        }
+        return 0;
+      } else { /* bridge matched, but it doesn't use transports. */
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(bridge);
+
+  *transport = NULL;
+  return 0;
+}
+
+/** Return a smartlist containing all the SOCKS arguments that we
+ *  should pass to the SOCKS proxy. */
+const smartlist_t *
+get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+{
+  bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr,
+                                                                    port,
+                                                                    NULL);
+  return bridge ? bridge->socks_args : NULL;
+}
+
+/** We need to ask <b>bridge</b> for its server descriptor. */
+static void
+launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge)
+{
+  const or_options_t *options = get_options();
+  circuit_guard_state_t *guard_state = NULL;
+
+  if (connection_get_by_type_addr_port_purpose(
+      CONN_TYPE_DIR, &bridge->addr, bridge->port,
+      DIR_PURPOSE_FETCH_SERVERDESC))
+    return; /* it's already on the way */
+
+  if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
+    download_status_mark_impossible(&bridge->fetch_status);
+    log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
+             safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
+    return;
+  }
+
+  /* Until we get a descriptor for the bridge, we only know one address for
+   * it. */
+  if (!fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
+                                            FIREWALL_OR_CONNECTION, 0, 0)) {
+    log_notice(LD_CONFIG, "Tried to fetch a descriptor directly from a "
+               "bridge, but that bridge is not reachable through our "
+               "firewall.");
+    return;
+  }
+
+  /* If we already have a node_t for this bridge, rewrite its address now. */
+  node_t *node = node_get_mutable_by_id(bridge->identity);
+  if (node) {
+    rewrite_node_address_for_bridge(bridge, node);
+  }
+
+  tor_addr_port_t bridge_addrport;
+  memcpy(&bridge_addrport.addr, &bridge->addr, sizeof(tor_addr_t));
+  bridge_addrport.port = bridge->port;
+
+  guard_state = get_guard_state_for_bridge_desc_fetch(bridge->identity);
+
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC);
+  directory_request_set_or_addr_port(req, &bridge_addrport);
+  directory_request_set_directory_id_digest(req, bridge->identity);
+  directory_request_set_router_purpose(req, ROUTER_PURPOSE_BRIDGE);
+  directory_request_set_resource(req, "authority.z");
+  if (guard_state) {
+    directory_request_set_guard_state(req, guard_state);
+  }
+  directory_initiate_request(req);
+  directory_request_free(req);
+}
+
+/** Fetching the bridge descriptor from the bridge authority returned a
+ * "not found". Fall back to trying a direct fetch. */
+void
+retry_bridge_descriptor_fetch_directly(const char *digest)
+{
+  bridge_info_t *bridge = find_bridge_by_digest(digest);
+  if (!bridge)
+    return; /* not found? oh well. */
+
+  launch_direct_bridge_descriptor_fetch(bridge);
+}
+
+/** For each bridge in our list for which we don't currently have a
+ * descriptor, fetch a new copy of its descriptor -- either directly
+ * from the bridge or via a bridge authority. */
+void
+fetch_bridge_descriptors(const or_options_t *options, time_t now)
+{
+  int num_bridge_auths = get_n_authorities(BRIDGE_DIRINFO);
+  int ask_bridge_directly;
+  int can_use_bridge_authority;
+
+  if (!bridge_list)
+    return;
+
+  /* If we still have unconfigured managed proxies, don't go and
+     connect to a bridge. */
+  if (pt_proxies_configuration_pending())
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      /* This resets the download status on first use */
+      if (!download_status_is_ready(&bridge->fetch_status, now))
+        continue; /* don't bother, no need to retry yet */
+      if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
+        download_status_mark_impossible(&bridge->fetch_status);
+        log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
+                 safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
+        continue;
+      }
+
+      /* schedule the next attempt
+       * we can't increment after a failure, because sometimes we use the
+       * bridge authority, and sometimes we use the bridge direct */
+      download_status_increment_attempt(
+                        &bridge->fetch_status,
+                        safe_str_client(fmt_and_decorate_addr(&bridge->addr)),
+                        now);
+
+      can_use_bridge_authority = !tor_digest_is_zero(bridge->identity) &&
+                                 num_bridge_auths;
+      ask_bridge_directly = !can_use_bridge_authority ||
+                            !options->UpdateBridgesFromAuthority;
+      log_debug(LD_DIR, "ask_bridge_directly=%d (%d, %d, %d)",
+                ask_bridge_directly, tor_digest_is_zero(bridge->identity),
+                !options->UpdateBridgesFromAuthority, !num_bridge_auths);
+
+      if (ask_bridge_directly &&
+          !fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
+                                                FIREWALL_OR_CONNECTION, 0,
+                                                0)) {
+        log_notice(LD_DIR, "Bridge at '%s' isn't reachable by our "
+                   "firewall policy. %s.",
+                   fmt_addrport(&bridge->addr, bridge->port),
+                   can_use_bridge_authority ?
+                     "Asking bridge authority instead" : "Skipping");
+        if (can_use_bridge_authority)
+          ask_bridge_directly = 0;
+        else
+          continue;
+      }
+
+      if (ask_bridge_directly) {
+        /* we need to ask the bridge itself for its descriptor. */
+        launch_direct_bridge_descriptor_fetch(bridge);
+      } else {
+        /* We have a digest and we want to ask an authority. We could
+         * combine all the requests into one, but that may give more
+         * hints to the bridge authority than we want to give. */
+        char resource[10 + HEX_DIGEST_LEN];
+        memcpy(resource, "fp/", 3);
+        base16_encode(resource+3, HEX_DIGEST_LEN+1,
+                      bridge->identity, DIGEST_LEN);
+        memcpy(resource+3+HEX_DIGEST_LEN, ".z", 3);
+        log_info(LD_DIR, "Fetching bridge info '%s' from bridge authority.",
+                 resource);
+        directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
+                ROUTER_PURPOSE_BRIDGE, resource, 0, DL_WANT_AUTHORITY);
+      }
+    }
+  SMARTLIST_FOREACH_END(bridge);
+}
+
+/** If our <b>bridge</b> is configured to be a different address than
+ * the bridge gives in <b>node</b>, rewrite the routerinfo
+ * we received to use the address we meant to use. Now we handle
+ * multihomed bridges better.
+ */
+static void
+rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node)
+{
+  /* XXXX move this function. */
+  /* XXXX overridden addresses should really live in the node_t, so that the
+   *   routerinfo_t and the microdesc_t can be immutable.  But we can only
+   *   do that safely if we know that no function that connects to an OR
+   *   does so through an address from any source other than node_get_addr().
+   */
+  tor_addr_t addr;
+  const or_options_t *options = get_options();
+
+  if (node->ri) {
+    routerinfo_t *ri = node->ri;
+    tor_addr_from_ipv4h(&addr, ri->addr);
+    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
+         bridge->port == ri->or_port) ||
+        (!tor_addr_compare(&bridge->addr, &ri->ipv6_addr, CMP_EXACT) &&
+         bridge->port == ri->ipv6_orport)) {
+      /* they match, so no need to do anything */
+    } else {
+      if (tor_addr_family(&bridge->addr) == AF_INET) {
+        ri->addr = tor_addr_to_ipv4h(&bridge->addr);
+        ri->or_port = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerinfo for '%s' to match configured "
+                 "address %s:%d.",
+                 ri->nickname, fmt_addr32(ri->addr), ri->or_port);
+      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
+        tor_addr_copy(&ri->ipv6_addr, &bridge->addr);
+        ri->ipv6_orport = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerinfo for '%s' to match configured "
+                 "address %s.",
+                 ri->nickname, fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
+      } else {
+        log_err(LD_BUG, "Address family not supported: %d.",
+                tor_addr_family(&bridge->addr));
+        return;
+      }
+    }
+
+    if (options->ClientPreferIPv6ORPort == -1) {
+      /* Mark which address to use based on which bridge_t we got. */
+      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
+                              !tor_addr_is_null(&node->ri->ipv6_addr));
+    } else {
+      /* Mark which address to use based on user preference */
+      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
+                              !tor_addr_is_null(&node->ri->ipv6_addr));
+    }
+
+    /* XXXipv6 we lack support for falling back to another address for
+       the same relay, warn the user */
+    if (!tor_addr_is_null(&ri->ipv6_addr)) {
+      tor_addr_port_t ap;
+      node_get_pref_orport(node, &ap);
+      log_notice(LD_CONFIG,
+                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
+                 "Will prefer using its %s address (%s) based on %s.",
+                 ri->nickname,
+                 node->ipv6_preferred ? "IPv6" : "IPv4",
+                 fmt_addrport(&ap.addr, ap.port),
+                 options->ClientPreferIPv6ORPort == -1 ?
+                 "the configured Bridge address" :
+                 "ClientPreferIPv6ORPort");
+    }
+  }
+  if (node->rs) {
+    routerstatus_t *rs = node->rs;
+    tor_addr_from_ipv4h(&addr, rs->addr);
+
+    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
+        bridge->port == rs->or_port) ||
+       (!tor_addr_compare(&bridge->addr, &rs->ipv6_addr, CMP_EXACT) &&
+        bridge->port == rs->ipv6_orport)) {
+      /* they match, so no need to do anything */
+    } else {
+      if (tor_addr_family(&bridge->addr) == AF_INET) {
+        rs->addr = tor_addr_to_ipv4h(&bridge->addr);
+        rs->or_port = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerstatus for '%s' to match "
+                 "configured address %s.",
+                 rs->nickname, fmt_addrport(&bridge->addr, rs->or_port));
+      /* set IPv6 preferences even if there is no ri */
+      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
+        tor_addr_copy(&rs->ipv6_addr, &bridge->addr);
+        rs->ipv6_orport = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerstatus for '%s' to match configured"
+                 " address %s.",
+                 rs->nickname, fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport));
+      } else {
+        log_err(LD_BUG, "Address family not supported: %d.",
+                tor_addr_family(&bridge->addr));
+        return;
+      }
+    }
+
+    if (options->ClientPreferIPv6ORPort == -1) {
+      /* Mark which address to use based on which bridge_t we got. */
+      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
+                              !tor_addr_is_null(&node->rs->ipv6_addr));
+    } else {
+      /* Mark which address to use based on user preference */
+      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
+                              !tor_addr_is_null(&node->rs->ipv6_addr));
+    }
+
+    /* XXXipv6 we lack support for falling back to another address for
+    the same relay, warn the user */
+    if (!tor_addr_is_null(&rs->ipv6_addr)) {
+      tor_addr_port_t ap;
+      node_get_pref_orport(node, &ap);
+      log_notice(LD_CONFIG,
+                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
+                 "Will prefer using its %s address (%s) based on %s.",
+                 rs->nickname,
+                 node->ipv6_preferred ? "IPv6" : "IPv4",
+                 fmt_addrport(&ap.addr, ap.port),
+                 options->ClientPreferIPv6ORPort == -1 ?
+                 "the configured Bridge address" :
+                 "ClientPreferIPv6ORPort");
+    }
+  }
+}
+
+/** We just learned a descriptor for a bridge. See if that
+ * digest is in our entry guard list, and add it if not. */
+void
+learned_bridge_descriptor(routerinfo_t *ri, int from_cache)
+{
+  tor_assert(ri);
+  tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE);
+  if (get_options()->UseBridges) {
+    /* Retry directory downloads whenever we get a bridge descriptor:
+     * - when bootstrapping, and
+     * - when we aren't sure if any of our bridges are reachable.
+     * Keep on retrying until we have at least one reachable bridge. */
+    int first = num_bridges_usable(0) < 1;
+    bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri);
+    time_t now = time(NULL);
+    router_set_status(ri->cache_info.identity_digest, 1);
+
+    if (bridge) { /* if we actually want to use this one */
+      node_t *node;
+      /* it's here; schedule its re-fetch for a long time from now. */
+      if (!from_cache) {
+        /* This schedules the re-fetch at a constant interval, which produces
+         * a pattern of bridge traffic. But it's better than trying all
+         * configured briges several times in the first few minutes. */
+        download_status_reset(&bridge->fetch_status);
+      }
+
+      node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+      tor_assert(node);
+      rewrite_node_address_for_bridge(bridge, node);
+      if (tor_digest_is_zero(bridge->identity)) {
+        memcpy(bridge->identity,ri->cache_info.identity_digest, DIGEST_LEN);
+        log_notice(LD_DIR, "Learned identity %s for bridge at %s:%d",
+                   hex_str(bridge->identity, DIGEST_LEN),
+                   fmt_and_decorate_addr(&bridge->addr),
+                   (int) bridge->port);
+      }
+      entry_guard_learned_bridge_identity(&bridge->addrport_configured,
+                              (const uint8_t*)ri->cache_info.identity_digest);
+
+      log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname,
+                 from_cache ? "cached" : "fresh", router_describe(ri));
+      /* If we didn't have a reachable bridge before this one, try directory
+       * documents again. */
+      if (first) {
+        routerlist_retry_directory_downloads(now);
+      }
+    }
+  }
+}
+
+/** Return a smartlist containing all bridge identity digests */
+MOCK_IMPL(smartlist_t *,
+list_bridge_identities, (void))
+{
+  smartlist_t *result = NULL;
+  char *digest_tmp;
+
+  if (get_options()->UseBridges && bridge_list) {
+    result = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+      digest_tmp = tor_malloc(DIGEST_LEN);
+      memcpy(digest_tmp, b->identity, DIGEST_LEN);
+      smartlist_add(result, digest_tmp);
+    } SMARTLIST_FOREACH_END(b);
+  }
+
+  return result;
+}
+
+/** Get the download status for a bridge descriptor given its identity */
+MOCK_IMPL(download_status_t *,
+get_bridge_dl_status_by_id, (const char *digest))
+{
+  download_status_t *dl = NULL;
+
+  if (digest && get_options()->UseBridges && bridge_list) {
+    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+      if (tor_memeq(digest, b->identity, DIGEST_LEN)) {
+        dl = &(b->fetch_status);
+        break;
+      }
+    } SMARTLIST_FOREACH_END(b);
+  }
+
+  return dl;
+}
+
+/** Release all storage held in bridges.c */
+void
+bridges_free_all(void)
+{
+  clear_bridge_list();
+  smartlist_free(bridge_list);
+  bridge_list = NULL;
+}
+
diff --git a/src/feature/client/bridges.h b/src/feature/client/bridges.h
new file mode 100644
index 0000000000..70588c1b91
--- /dev/null
+++ b/src/feature/client/bridges.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bridges.h
+ * \brief Header file for circuitbuild.c.
+ **/
+
+#ifndef TOR_BRIDGES_H
+#define TOR_BRIDGES_H
+
+struct bridge_line_t;
+struct ed25519_public_key_t;
+
+/* Opaque handle to a configured bridge */
+typedef struct bridge_info_t bridge_info_t;
+
+void mark_bridge_list(void);
+void sweep_bridge_list(void);
+const smartlist_t *bridge_list_get(void);
+const uint8_t *bridge_get_rsa_id_digest(const bridge_info_t *bridge);
+const tor_addr_port_t * bridge_get_addr_port(const bridge_info_t *bridge);
+bridge_info_t *get_configured_bridge_by_addr_port_digest(
+                                          const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest);
+bridge_info_t *get_configured_bridge_by_exact_addr_port_digest(
+                                          const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest);
+
+int addr_is_a_configured_bridge(const tor_addr_t *addr, uint16_t port,
+                                const char *digest);
+int extend_info_is_a_configured_bridge(const extend_info_t *ei);
+int routerinfo_is_a_configured_bridge(const routerinfo_t *ri);
+int node_is_a_configured_bridge(const node_t *node);
+void learned_router_identity(const tor_addr_t *addr, uint16_t port,
+                             const char *digest,
+                             const struct ed25519_public_key_t *ed_id);
+
+void bridge_add_from_config(struct bridge_line_t *bridge_line);
+void retry_bridge_descriptor_fetch_directly(const char *digest);
+void fetch_bridge_descriptors(const or_options_t *options, time_t now);
+void learned_bridge_descriptor(routerinfo_t *ri, int from_cache);
+const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr,
+                                                     uint16_t port);
+
+int any_bridges_dont_support_microdescriptors(void);
+
+const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr,
+                                                   uint16_t port);
+struct transport_t;
+int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
+                                      const struct transport_t **transport);
+
+MOCK_DECL(int, transport_is_needed, (const char *transport_name));
+int validate_pluggable_transports_config(void);
+
+MOCK_DECL(smartlist_t *, list_bridge_identities, (void));
+MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id,
+          (const char *digest));
+
+void bridges_free_all(void);
+
+#ifdef TOR_BRIDGES_PRIVATE
+STATIC void clear_bridge_list(void);
+STATIC bridge_info_t *find_bridge_by_digest(const char *digest);
+STATIC bridge_info_t *get_configured_bridge_by_orports_digest(
+                                                  const char *digest,
+                                                  const smartlist_t *orports);
+STATIC void bridge_resolve_conflicts(const tor_addr_t *addr,
+                                     uint16_t port,
+                                     const char *digest,
+                                     const char *transport_name);
+#endif /* defined(TOR_BRIDGES_PRIVATE) */
+
+#endif /* !defined(TOR_BRIDGES_H) */
diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c
new file mode 100644
index 0000000000..32b3212d3f
--- /dev/null
+++ b/src/feature/client/circpathbias.c
@@ -0,0 +1,1578 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circpathbias.c
+ *
+ * \brief Code to track success/failure rates of circuits built through
+ * different tor nodes, in an attempt to detect attacks where
+ * an attacker deliberately causes circuits to fail until the client
+ * choses a path they like.
+ *
+ * This code is currently configured in a warning-only mode, though false
+ * positives appear to be rare in practice.  There is also support for
+ * disabling really bad guards, but it's quite experimental and may have bad
+ * anonymity effects.
+ *
+ * The information here is associated with the entry_guard_t object for
+ * each guard, and stored persistently in the state file.
+ */
+
+#include "or/or.h"
+#include "or/channel.h"
+#include "or/circpathbias.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/circuitstats.h"
+#include "or/connection_edge.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "or/entrynodes.h"
+#include "or/networkstatus.h"
+#include "or/relay.h"
+#include "lib/math/fp.h"
+#include "lib/math/laplace.h"
+
+#include "or/cell_st.h"
+#include "or/cpath_build_state_st.h"
+#include "or/crypt_path_st.h"
+#include "or/extend_info_st.h"
+#include "or/origin_circuit_st.h"
+
+static void pathbias_count_successful_close(origin_circuit_t *circ);
+static void pathbias_count_collapse(origin_circuit_t *circ);
+static void pathbias_count_use_failed(origin_circuit_t *circ);
+static void pathbias_measure_use_rate(entry_guard_t *guard);
+static void pathbias_measure_close_rate(entry_guard_t *guard);
+static void pathbias_scale_use_rates(entry_guard_t *guard);
+static void pathbias_scale_close_rates(entry_guard_t *guard);
+static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard);
+
+/** Increment the number of times we successfully extended a circuit to
+ * <b>guard</b>, first checking if the failure rate is high enough that
+ * we should eliminate the guard. Return -1 if the guard looks no good;
+ * return 0 if the guard looks fine.
+ */
+static int
+entry_guard_inc_circ_attempt_count(entry_guard_t *guard)
+{
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  entry_guards_changed();
+
+  pathbias_measure_close_rate(guard);
+
+  if (pb->path_bias_disabled)
+    return -1;
+
+  pathbias_scale_close_rates(guard);
+  pb->circ_attempts++;
+
+  log_info(LD_CIRC, "Got success count %f/%f for guard %s",
+           pb->circ_successes, pb->circ_attempts,
+           entry_guard_describe(guard));
+  return 0;
+}
+
+/** The minimum number of circuit attempts before we start
+  * thinking about warning about path bias and dropping guards */
+static int
+pathbias_get_min_circs(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_MIN_CIRC 150
+  if (options->PathBiasCircThreshold >= 5)
+    return options->PathBiasCircThreshold;
+  else
+    return networkstatus_get_param(NULL, "pb_mincircs",
+                                   DFLT_PATH_BIAS_MIN_CIRC,
+                                   5, INT32_MAX);
+}
+
+/** The circuit success rate below which we issue a notice */
+static double
+pathbias_get_notice_rate(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_NOTICE_PCT 70
+  if (options->PathBiasNoticeRate >= 0.0)
+    return options->PathBiasNoticeRate;
+  else
+    return networkstatus_get_param(NULL, "pb_noticepct",
+                                   DFLT_PATH_BIAS_NOTICE_PCT, 0, 100)/100.0;
+}
+
+/** The circuit success rate below which we issue a warn */
+static double
+pathbias_get_warn_rate(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_WARN_PCT 50
+  if (options->PathBiasWarnRate >= 0.0)
+    return options->PathBiasWarnRate;
+  else
+    return networkstatus_get_param(NULL, "pb_warnpct",
+                                   DFLT_PATH_BIAS_WARN_PCT, 0, 100)/100.0;
+}
+
+/* XXXX I'd like to have this be static again, but entrynodes.c needs it. */
+/**
+ * The extreme rate is the rate at which we would drop the guard,
+ * if pb_dropguard is also set. Otherwise we just warn.
+ */
+double
+pathbias_get_extreme_rate(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_EXTREME_PCT 30
+  if (options->PathBiasExtremeRate >= 0.0)
+    return options->PathBiasExtremeRate;
+  else
+    return networkstatus_get_param(NULL, "pb_extremepct",
+                                   DFLT_PATH_BIAS_EXTREME_PCT, 0, 100)/100.0;
+}
+
+/* XXXX I'd like to have this be static again, but entrynodes.c needs it. */
+/**
+ * If 1, we actually disable use of guards that fall below
+ * the extreme_pct.
+ */
+int
+pathbias_get_dropguards(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_DROP_GUARDS 0
+  if (options->PathBiasDropGuards >= 0)
+    return options->PathBiasDropGuards;
+  else
+    return networkstatus_get_param(NULL, "pb_dropguards",
+                                   DFLT_PATH_BIAS_DROP_GUARDS, 0, 1);
+}
+
+/**
+ * This is the number of circuits at which we scale our
+ * counts by mult_factor/scale_factor. Note, this count is
+ * not exact, as we only perform the scaling in the event
+ * of no integer truncation.
+ */
+static int
+pathbias_get_scale_threshold(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_SCALE_THRESHOLD 300
+  if (options->PathBiasScaleThreshold >= 10)
+    return options->PathBiasScaleThreshold;
+  else
+    return networkstatus_get_param(NULL, "pb_scalecircs",
+                                   DFLT_PATH_BIAS_SCALE_THRESHOLD, 10,
+                                   INT32_MAX);
+}
+
+/**
+ * Compute the path bias scaling ratio from the consensus
+ * parameters pb_multfactor/pb_scalefactor.
+ *
+ * Returns a value in (0, 1.0] which we multiply our pathbias
+ * counts with to scale them down.
+ */
+static double
+pathbias_get_scale_ratio(const or_options_t *options)
+{
+  /*
+   * The scale factor is the denominator for our scaling
+   * of circuit counts for our path bias window.
+   *
+   * Note that our use of doubles for the path bias state
+   * file means that powers of 2 work best here.
+   */
+  int denominator = networkstatus_get_param(NULL, "pb_scalefactor",
+                              2, 2, INT32_MAX);
+  (void) options;
+  /**
+   * The mult factor is the numerator for our scaling
+   * of circuit counts for our path bias window. It
+   * allows us to scale by fractions.
+   */
+  return networkstatus_get_param(NULL, "pb_multfactor",
+                              1, 1, denominator)/((double)denominator);
+}
+
+/** The minimum number of circuit usage attempts before we start
+  * thinking about warning about path use bias and dropping guards */
+static int
+pathbias_get_min_use(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_MIN_USE 20
+  if (options->PathBiasUseThreshold >= 3)
+    return options->PathBiasUseThreshold;
+  else
+    return networkstatus_get_param(NULL, "pb_minuse",
+                                   DFLT_PATH_BIAS_MIN_USE,
+                                   3, INT32_MAX);
+}
+
+/** The circuit use success rate below which we issue a notice */
+static double
+pathbias_get_notice_use_rate(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_NOTICE_USE_PCT 80
+  if (options->PathBiasNoticeUseRate >= 0.0)
+    return options->PathBiasNoticeUseRate;
+  else
+    return networkstatus_get_param(NULL, "pb_noticeusepct",
+                                   DFLT_PATH_BIAS_NOTICE_USE_PCT,
+                                   0, 100)/100.0;
+}
+
+/**
+ * The extreme use rate is the rate at which we would drop the guard,
+ * if pb_dropguard is also set. Otherwise we just warn.
+ */
+double
+pathbias_get_extreme_use_rate(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_EXTREME_USE_PCT 60
+  if (options->PathBiasExtremeUseRate >= 0.0)
+    return options->PathBiasExtremeUseRate;
+  else
+    return networkstatus_get_param(NULL, "pb_extremeusepct",
+                                   DFLT_PATH_BIAS_EXTREME_USE_PCT,
+                                   0, 100)/100.0;
+}
+
+/**
+ * This is the number of circuits at which we scale our
+ * use counts by mult_factor/scale_factor. Note, this count is
+ * not exact, as we only perform the scaling in the event
+ * of no integer truncation.
+ */
+static int
+pathbias_get_scale_use_threshold(const or_options_t *options)
+{
+#define DFLT_PATH_BIAS_SCALE_USE_THRESHOLD 100
+  if (options->PathBiasScaleUseThreshold >= 10)
+    return options->PathBiasScaleUseThreshold;
+  else
+    return networkstatus_get_param(NULL, "pb_scaleuse",
+                                   DFLT_PATH_BIAS_SCALE_USE_THRESHOLD,
+                                   10, INT32_MAX);
+}
+
+/**
+ * Convert a Guard's path state to string.
+ */
+const char *
+pathbias_state_to_string(path_state_t state)
+{
+  switch (state) {
+    case PATH_STATE_NEW_CIRC:
+      return "new";
+    case PATH_STATE_BUILD_ATTEMPTED:
+      return "build attempted";
+    case PATH_STATE_BUILD_SUCCEEDED:
+      return "build succeeded";
+    case PATH_STATE_USE_ATTEMPTED:
+      return "use attempted";
+    case PATH_STATE_USE_SUCCEEDED:
+      return "use succeeded";
+    case PATH_STATE_USE_FAILED:
+      return "use failed";
+    case PATH_STATE_ALREADY_COUNTED:
+      return "already counted";
+  }
+
+  return "unknown";
+}
+
+/**
+ * This function decides if a circuit has progressed far enough to count
+ * as a circuit "attempt". As long as end-to-end tagging is possible,
+ * we assume the adversary will use it over hop-to-hop failure. Therefore,
+ * we only need to account bias for the last hop. This should make us
+ * much more resilient to ambient circuit failure, and also make that
+ * failure easier to measure (we only need to measure Exit failure rates).
+ */
+static int
+pathbias_is_new_circ_attempt(origin_circuit_t *circ)
+{
+#define N2N_TAGGING_IS_POSSIBLE
+#ifdef N2N_TAGGING_IS_POSSIBLE
+  /* cpath is a circular list. We want circs with more than one hop,
+   * and the second hop must be waiting for keys still (it's just
+   * about to get them). */
+  return circ->cpath &&
+         circ->cpath->next != circ->cpath &&
+         circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS;
+#else /* !(defined(N2N_TAGGING_IS_POSSIBLE)) */
+  /* If tagging attacks are no longer possible, we probably want to
+   * count bias from the first hop. However, one could argue that
+   * timing-based tagging is still more useful than per-hop failure.
+   * In which case, we'd never want to use this.
+   */
+  return circ->cpath &&
+         circ->cpath->state == CPATH_STATE_AWAITING_KEYS;
+#endif /* defined(N2N_TAGGING_IS_POSSIBLE) */
+}
+
+/**
+ * Decide if the path bias code should count a circuit.
+ *
+ * @returns 1 if we should count it, 0 otherwise.
+ */
+static int
+pathbias_should_count(origin_circuit_t *circ)
+{
+#define PATHBIAS_COUNT_INTERVAL (600)
+  static ratelim_t count_limit =
+    RATELIM_INIT(PATHBIAS_COUNT_INTERVAL);
+  char *rate_msg = NULL;
+
+  /* We can't do path bias accounting without entry guards.
+   * Testing and controller circuits also have no guards.
+   *
+   * We also don't count server-side rends, because their
+   * endpoint could be chosen maliciously.
+   * Similarly, we can't count client-side intro attempts,
+   * because clients can be manipulated into connecting to
+   * malicious intro points. */
+  if (get_options()->UseEntryGuards == 0 ||
+          circ->base_.purpose == CIRCUIT_PURPOSE_TESTING ||
+          circ->base_.purpose == CIRCUIT_PURPOSE_CONTROLLER ||
+          circ->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND ||
+          circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED ||
+          (circ->base_.purpose >= CIRCUIT_PURPOSE_C_INTRODUCING &&
+           circ->base_.purpose <= CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) {
+
+    /* Check to see if the shouldcount result has changed due to a
+     * unexpected purpose change that would affect our results.
+     *
+     * The reason we check the path state too here is because for the
+     * cannibalized versions of these purposes, we count them as successful
+     * before their purpose change.
+     */
+    if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED
+            && circ->path_state != PATH_STATE_ALREADY_COUNTED) {
+      log_info(LD_BUG,
+               "Circuit %d is now being ignored despite being counted "
+               "in the past. Purpose is %s, path state is %s",
+               circ->global_identifier,
+               circuit_purpose_to_string(circ->base_.purpose),
+               pathbias_state_to_string(circ->path_state));
+    }
+    circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED;
+    return 0;
+  }
+
+  /* Completely ignore one hop circuits */
+  if (circ->build_state->onehop_tunnel ||
+      circ->build_state->desired_path_len == 1) {
+    /* Check for inconsistency */
+    if (circ->build_state->desired_path_len != 1 ||
+        !circ->build_state->onehop_tunnel) {
+      if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) {
+        log_info(LD_BUG,
+               "One-hop circuit has length %d. Path state is %s. "
+               "Circuit is a %s currently %s.%s",
+               circ->build_state->desired_path_len,
+               pathbias_state_to_string(circ->path_state),
+               circuit_purpose_to_string(circ->base_.purpose),
+               circuit_state_to_string(circ->base_.state),
+               rate_msg);
+        tor_free(rate_msg);
+      }
+      tor_fragile_assert();
+    }
+
+    /* Check to see if the shouldcount result has changed due to a
+     * unexpected change that would affect our results */
+    if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED) {
+      log_info(LD_BUG,
+               "One-hop circuit %d is now being ignored despite being counted "
+               "in the past. Purpose is %s, path state is %s",
+               circ->global_identifier,
+               circuit_purpose_to_string(circ->base_.purpose),
+               pathbias_state_to_string(circ->path_state));
+    }
+    circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED;
+    return 0;
+  }
+
+  /* Check to see if the shouldcount result has changed due to a
+   * unexpected purpose change that would affect our results */
+  if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) {
+      log_info(LD_BUG,
+              "Circuit %d is now being counted despite being ignored "
+              "in the past. Purpose is %s, path state is %s",
+              circ->global_identifier,
+              circuit_purpose_to_string(circ->base_.purpose),
+              pathbias_state_to_string(circ->path_state));
+  }
+  circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED;
+
+  return 1;
+}
+
+/**
+ * Check our circuit state to see if this is a successful circuit attempt.
+ * If so, record it in the current guard's path bias circ_attempt count.
+ *
+ * Also check for several potential error cases for bug #6475.
+ */
+int
+pathbias_count_build_attempt(origin_circuit_t *circ)
+{
+#define CIRC_ATTEMPT_NOTICE_INTERVAL (600)
+  static ratelim_t circ_attempt_notice_limit =
+    RATELIM_INIT(CIRC_ATTEMPT_NOTICE_INTERVAL);
+  char *rate_msg = NULL;
+
+  if (!pathbias_should_count(circ)) {
+    return 0;
+  }
+
+  if (pathbias_is_new_circ_attempt(circ)) {
+    /* Help track down the real cause of bug #6475: */
+    if (circ->has_opened && circ->path_state != PATH_STATE_BUILD_ATTEMPTED) {
+      if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
+                                     approx_time()))) {
+        log_info(LD_BUG,
+                "Opened circuit is in strange path state %s. "
+                "Circuit is a %s currently %s.%s",
+                pathbias_state_to_string(circ->path_state),
+                circuit_purpose_to_string(circ->base_.purpose),
+                circuit_state_to_string(circ->base_.state),
+                rate_msg);
+        tor_free(rate_msg);
+      }
+    }
+
+    /* Don't re-count cannibalized circs.. */
+    if (!circ->has_opened) {
+      entry_guard_t *guard = NULL;
+
+      if (circ->cpath && circ->cpath->extend_info) {
+        guard = entry_guard_get_by_id_digest(
+                  circ->cpath->extend_info->identity_digest);
+      } else if (circ->base_.n_chan) {
+        guard =
+          entry_guard_get_by_id_digest(circ->base_.n_chan->identity_digest);
+      }
+
+      if (guard) {
+        if (circ->path_state == PATH_STATE_NEW_CIRC) {
+          circ->path_state = PATH_STATE_BUILD_ATTEMPTED;
+
+          if (entry_guard_inc_circ_attempt_count(guard) < 0) {
+            /* Bogus guard; we already warned. */
+            return -END_CIRC_REASON_TORPROTOCOL;
+          }
+        } else {
+          if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
+                  approx_time()))) {
+            log_info(LD_BUG,
+                   "Unopened circuit has strange path state %s. "
+                   "Circuit is a %s currently %s.%s",
+                   pathbias_state_to_string(circ->path_state),
+                   circuit_purpose_to_string(circ->base_.purpose),
+                   circuit_state_to_string(circ->base_.state),
+                   rate_msg);
+            tor_free(rate_msg);
+          }
+        }
+      } else {
+        if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
+                approx_time()))) {
+          log_info(LD_CIRC,
+              "Unopened circuit has no known guard. "
+              "Circuit is a %s currently %s.%s",
+              circuit_purpose_to_string(circ->base_.purpose),
+              circuit_state_to_string(circ->base_.state),
+              rate_msg);
+          tor_free(rate_msg);
+        }
+      }
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Check our circuit state to see if this is a successful circuit
+ * completion. If so, record it in the current guard's path bias
+ * success count.
+ *
+ * Also check for several potential error cases for bug #6475.
+ */
+void
+pathbias_count_build_success(origin_circuit_t *circ)
+{
+#define SUCCESS_NOTICE_INTERVAL (600)
+  static ratelim_t success_notice_limit =
+    RATELIM_INIT(SUCCESS_NOTICE_INTERVAL);
+  char *rate_msg = NULL;
+  entry_guard_t *guard = NULL;
+
+  if (!pathbias_should_count(circ)) {
+    return;
+  }
+
+  /* Don't count cannibalized/reused circs for path bias
+   * "build" success, since they get counted under "use" success. */
+  if (!circ->has_opened) {
+    if (circ->cpath && circ->cpath->extend_info) {
+      guard = entry_guard_get_by_id_digest(
+                circ->cpath->extend_info->identity_digest);
+    }
+
+    if (guard) {
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+      if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) {
+        circ->path_state = PATH_STATE_BUILD_SUCCEEDED;
+        pb->circ_successes++;
+        entry_guards_changed();
+
+        log_info(LD_CIRC, "Got success count %f/%f for guard %s",
+                 pb->circ_successes, pb->circ_attempts,
+                 entry_guard_describe(guard));
+      } else {
+        if ((rate_msg = rate_limit_log(&success_notice_limit,
+                approx_time()))) {
+          log_info(LD_BUG,
+              "Succeeded circuit is in strange path state %s. "
+              "Circuit is a %s currently %s.%s",
+              pathbias_state_to_string(circ->path_state),
+              circuit_purpose_to_string(circ->base_.purpose),
+              circuit_state_to_string(circ->base_.state),
+              rate_msg);
+          tor_free(rate_msg);
+        }
+      }
+
+      if (pb->circ_attempts < pb->circ_successes) {
+        log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) "
+                 "for guard %s",
+                 pb->circ_successes, pb->circ_attempts,
+                 entry_guard_describe(guard));
+      }
+    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
+     * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here.
+     * No need to log that case. */
+    } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+      if ((rate_msg = rate_limit_log(&success_notice_limit,
+              approx_time()))) {
+        log_info(LD_CIRC,
+            "Completed circuit has no known guard. "
+            "Circuit is a %s currently %s.%s",
+            circuit_purpose_to_string(circ->base_.purpose),
+            circuit_state_to_string(circ->base_.state),
+            rate_msg);
+        tor_free(rate_msg);
+      }
+    }
+  } else {
+    if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) {
+      if ((rate_msg = rate_limit_log(&success_notice_limit,
+              approx_time()))) {
+        log_info(LD_BUG,
+            "Opened circuit is in strange path state %s. "
+            "Circuit is a %s currently %s.%s",
+            pathbias_state_to_string(circ->path_state),
+            circuit_purpose_to_string(circ->base_.purpose),
+            circuit_state_to_string(circ->base_.state),
+            rate_msg);
+        tor_free(rate_msg);
+      }
+    }
+  }
+}
+
+/**
+ * Record an attempt to use a circuit. Changes the circuit's
+ * path state and update its guard's usage counter.
+ *
+ * Used for path bias usage accounting.
+ */
+void
+pathbias_count_use_attempt(origin_circuit_t *circ)
+{
+  if (!pathbias_should_count(circ)) {
+    return;
+  }
+
+  if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) {
+    log_notice(LD_BUG,
+        "Used circuit is in strange path state %s. "
+        "Circuit is a %s currently %s.",
+        pathbias_state_to_string(circ->path_state),
+        circuit_purpose_to_string(circ->base_.purpose),
+        circuit_state_to_string(circ->base_.state));
+  } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) {
+    entry_guard_t *guard = entry_guard_get_by_id_digest(
+                circ->cpath->extend_info->identity_digest);
+    if (guard) {
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+      pathbias_measure_use_rate(guard);
+      pathbias_scale_use_rates(guard);
+      pb->use_attempts++;
+      entry_guards_changed();
+
+      log_debug(LD_CIRC,
+               "Marked circuit %d (%f/%f) as used for guard %s.",
+               circ->global_identifier,
+               pb->use_successes, pb->use_attempts,
+               entry_guard_describe(guard));
+    }
+
+    circ->path_state = PATH_STATE_USE_ATTEMPTED;
+  } else {
+    /* Harmless but educational log message */
+    log_info(LD_CIRC,
+        "Used circuit %d is already in path state %s. "
+        "Circuit is a %s currently %s.",
+        circ->global_identifier,
+        pathbias_state_to_string(circ->path_state),
+        circuit_purpose_to_string(circ->base_.purpose),
+        circuit_state_to_string(circ->base_.state));
+  }
+
+  return;
+}
+
+/**
+ * Check the circuit's path state is appropriate and mark it as
+ * successfully used. Used for path bias usage accounting.
+ *
+ * We don't actually increment the guard's counters until
+ * pathbias_check_close(), because the circuit can still transition
+ * back to PATH_STATE_USE_ATTEMPTED if a stream fails later (this
+ * is done so we can probe the circuit for liveness at close).
+ */
+void
+pathbias_mark_use_success(origin_circuit_t *circ)
+{
+  if (!pathbias_should_count(circ)) {
+    return;
+  }
+
+  if (circ->path_state < PATH_STATE_USE_ATTEMPTED) {
+    log_notice(LD_BUG,
+        "Used circuit %d is in strange path state %s. "
+        "Circuit is a %s currently %s.",
+        circ->global_identifier,
+        pathbias_state_to_string(circ->path_state),
+        circuit_purpose_to_string(circ->base_.purpose),
+        circuit_state_to_string(circ->base_.state));
+
+    pathbias_count_use_attempt(circ);
+  }
+
+  /* We don't do any accounting at the guard until actual circuit close */
+  circ->path_state = PATH_STATE_USE_SUCCEEDED;
+
+  return;
+}
+
+/**
+ * If a stream ever detatches from a circuit in a retriable way,
+ * we need to mark this circuit as still needing either another
+ * successful stream, or in need of a probe.
+ *
+ * An adversary could let the first stream request succeed (ie the
+ * resolve), but then tag and timeout the remainder (via cell
+ * dropping), forcing them on new circuits.
+ *
+ * Rolling back the state will cause us to probe such circuits, which
+ * should lead to probe failures in the event of such tagging due to
+ * either unrecognized cells coming in while we wait for the probe,
+ * or the cipher state getting out of sync in the case of dropped cells.
+ */
+void
+pathbias_mark_use_rollback(origin_circuit_t *circ)
+{
+  if (circ->path_state == PATH_STATE_USE_SUCCEEDED) {
+    log_info(LD_CIRC,
+             "Rolling back pathbias use state to 'attempted' for detached "
+             "circuit %d", circ->global_identifier);
+    circ->path_state = PATH_STATE_USE_ATTEMPTED;
+  }
+}
+
+/**
+ * Actually count a circuit success towards a guard's usage counters
+ * if the path state is appropriate.
+ */
+static void
+pathbias_count_use_success(origin_circuit_t *circ)
+{
+  entry_guard_t *guard;
+
+  if (!pathbias_should_count(circ)) {
+    return;
+  }
+
+  if (circ->path_state != PATH_STATE_USE_SUCCEEDED) {
+    log_notice(LD_BUG,
+        "Successfully used circuit %d is in strange path state %s. "
+        "Circuit is a %s currently %s.",
+        circ->global_identifier,
+        pathbias_state_to_string(circ->path_state),
+        circuit_purpose_to_string(circ->base_.purpose),
+        circuit_state_to_string(circ->base_.state));
+  } else {
+    guard = entry_guard_get_by_id_digest(
+                circ->cpath->extend_info->identity_digest);
+    if (guard) {
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+      pb->use_successes++;
+      entry_guards_changed();
+
+      if (pb->use_attempts < pb->use_successes) {
+        log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) "
+                 "for guard %s",
+                 pb->use_successes, pb->use_attempts,
+                 entry_guard_describe(guard));
+      }
+
+      log_debug(LD_CIRC,
+                "Marked circuit %d (%f/%f) as used successfully for guard %s",
+                circ->global_identifier, pb->use_successes,
+                pb->use_attempts,
+                entry_guard_describe(guard));
+    }
+  }
+
+  return;
+}
+
+/**
+ * Send a probe down a circuit that the client attempted to use,
+ * but for which the stream timed out/failed. The probe is a
+ * RELAY_BEGIN cell with a 0.a.b.c destination address, which
+ * the exit will reject and reply back, echoing that address.
+ *
+ * The reason for such probes is because it is possible to bias
+ * a user's paths simply by causing timeouts, and these timeouts
+ * are not possible to differentiate from unresponsive servers.
+ *
+ * The probe is sent at the end of the circuit lifetime for two
+ * reasons: to prevent cryptographic taggers from being able to
+ * drop cells to cause timeouts, and to prevent easy recognition
+ * of probes before any real client traffic happens.
+ *
+ * Returns -1 if we couldn't probe, 0 otherwise.
+ */
+static int
+pathbias_send_usable_probe(circuit_t *circ)
+{
+  /* Based on connection_ap_handshake_send_begin() */
+  char payload[CELL_PAYLOAD_SIZE];
+  int payload_len;
+  origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+  crypt_path_t *cpath_layer = NULL;
+  char *probe_nonce = NULL;
+
+  tor_assert(ocirc);
+
+  cpath_layer = ocirc->cpath->prev;
+
+  if (cpath_layer->state != CPATH_STATE_OPEN) {
+    /* This can happen for cannibalized circuits. Their
+     * last hop isn't yet open */
+    log_info(LD_CIRC,
+             "Got pathbias probe request for unopened circuit %d. "
+             "Opened %d, len %d", ocirc->global_identifier,
+             ocirc->has_opened, ocirc->build_state->desired_path_len);
+    return -1;
+  }
+
+  /* We already went down this road. */
+  if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING &&
+      ocirc->pathbias_probe_id) {
+    log_info(LD_CIRC,
+             "Got pathbias probe request for circuit %d with "
+             "outstanding probe", ocirc->global_identifier);
+    return -1;
+  }
+
+  /* Can't probe if the channel isn't open */
+  if (circ->n_chan == NULL ||
+      (!CHANNEL_IS_OPEN(circ->n_chan)
+       && !CHANNEL_IS_MAINT(circ->n_chan))) {
+    log_info(LD_CIRC,
+             "Skipping pathbias probe for circuit %d: Channel is not open.",
+             ocirc->global_identifier);
+    return -1;
+  }
+
+  circuit_change_purpose(circ, CIRCUIT_PURPOSE_PATH_BIAS_TESTING);
+
+  /* Update timestamp for when circuit_expire_building() should kill us */
+  tor_gettimeofday(&circ->timestamp_began);
+
+  /* Generate a random address for the nonce */
+  crypto_rand((char*)&ocirc->pathbias_probe_nonce,
+              sizeof(ocirc->pathbias_probe_nonce));
+  ocirc->pathbias_probe_nonce &= 0x00ffffff;
+  probe_nonce = tor_dup_ip(ocirc->pathbias_probe_nonce);
+
+  tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:25", probe_nonce);
+  payload_len = (int)strlen(payload)+1;
+
+  // XXX: need this? Can we assume ipv4 will always be supported?
+  // If not, how do we tell?
+  //if (payload_len <= RELAY_PAYLOAD_SIZE - 4 && edge_conn->begincell_flags) {
+  //  set_uint32(payload + payload_len, htonl(edge_conn->begincell_flags));
+  //  payload_len += 4;
+  //}
+
+  /* Generate+Store stream id, make sure it's non-zero */
+  ocirc->pathbias_probe_id = get_unique_stream_id_by_circ(ocirc);
+
+  if (ocirc->pathbias_probe_id==0) {
+    log_warn(LD_CIRC,
+             "Ran out of stream IDs on circuit %u during "
+             "pathbias probe attempt.", ocirc->global_identifier);
+    tor_free(probe_nonce);
+    return -1;
+  }
+
+  log_info(LD_CIRC,
+           "Sending pathbias testing cell to %s:25 on stream %d for circ %d.",
+           probe_nonce, ocirc->pathbias_probe_id, ocirc->global_identifier);
+  tor_free(probe_nonce);
+
+  /* Send a test relay cell */
+  if (relay_send_command_from_edge(ocirc->pathbias_probe_id, circ,
+                               RELAY_COMMAND_BEGIN, payload,
+                               payload_len, cpath_layer) < 0) {
+    log_notice(LD_CIRC,
+               "Failed to send pathbias probe cell on circuit %d.",
+               ocirc->global_identifier);
+    return -1;
+  }
+
+  /* Mark it freshly dirty so it doesn't get expired in the meantime */
+  circ->timestamp_dirty = time(NULL);
+
+  return 0;
+}
+
+/**
+ * Check the response to a pathbias probe, to ensure the
+ * cell is recognized and the nonce and other probe
+ * characteristics are as expected.
+ *
+ * If the response is valid, return 0. Otherwise return < 0.
+ */
+int
+pathbias_check_probe_response(circuit_t *circ, const cell_t *cell)
+{
+  /* Based on connection_edge_process_relay_cell() */
+  relay_header_t rh;
+  int reason;
+  uint32_t ipv4_host;
+  origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+  tor_assert(cell);
+  tor_assert(ocirc);
+  tor_assert(circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING);
+
+  relay_header_unpack(&rh, cell->payload);
+
+  reason = rh.length > 0 ?
+        get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC;
+
+  if (rh.command == RELAY_COMMAND_END &&
+      reason == END_STREAM_REASON_EXITPOLICY &&
+      ocirc->pathbias_probe_id == rh.stream_id) {
+
+    /* Check length+extract host: It is in network order after the reason code.
+     * See connection_edge_end(). */
+    if (rh.length < 9) { /* reason+ipv4+dns_ttl */
+      log_notice(LD_PROTOCOL,
+             "Short path bias probe response length field (%d).", rh.length);
+      return - END_CIRC_REASON_TORPROTOCOL;
+    }
+
+    ipv4_host = ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+1));
+
+    /* Check nonce */
+    if (ipv4_host == ocirc->pathbias_probe_nonce) {
+      pathbias_mark_use_success(ocirc);
+      circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
+      log_info(LD_CIRC,
+               "Got valid path bias probe back for circ %d, stream %d.",
+               ocirc->global_identifier, ocirc->pathbias_probe_id);
+      return 0;
+    } else {
+      log_notice(LD_CIRC,
+               "Got strange probe value 0x%x vs 0x%x back for circ %d, "
+               "stream %d.", ipv4_host, ocirc->pathbias_probe_nonce,
+               ocirc->global_identifier, ocirc->pathbias_probe_id);
+      return -1;
+    }
+  }
+  log_info(LD_CIRC,
+             "Got another cell back back on pathbias probe circuit %d: "
+             "Command: %d, Reason: %d, Stream-id: %d",
+             ocirc->global_identifier, rh.command, reason, rh.stream_id);
+  return -1;
+}
+
+/**
+ * Check if a circuit was used and/or closed successfully.
+ *
+ * If we attempted to use the circuit to carry a stream but failed
+ * for whatever reason, or if the circuit mysteriously died before
+ * we could attach any streams, record these two cases.
+ *
+ * If we *have* successfully used the circuit, or it appears to
+ * have been closed by us locally, count it as a success.
+ *
+ * Returns 0 if we're done making decisions with the circ,
+ * or -1 if we want to probe it first.
+ */
+int
+pathbias_check_close(origin_circuit_t *ocirc, int reason)
+{
+  circuit_t *circ = &ocirc->base_;
+
+  if (!pathbias_should_count(ocirc)) {
+    return 0;
+  }
+
+  switch (ocirc->path_state) {
+    /* If the circuit was closed after building, but before use, we need
+     * to ensure we were the ones who tried to close it (and not a remote
+     * actor). */
+    case PATH_STATE_BUILD_SUCCEEDED:
+      if (reason & END_CIRC_REASON_FLAG_REMOTE) {
+        /* Remote circ close reasons on an unused circuit all could be bias */
+        log_info(LD_CIRC,
+            "Circuit %d remote-closed without successful use for reason %d. "
+            "Circuit purpose %d currently %d,%s. Len %d.",
+            ocirc->global_identifier,
+            reason, circ->purpose, ocirc->has_opened,
+            circuit_state_to_string(circ->state),
+            ocirc->build_state->desired_path_len);
+        pathbias_count_collapse(ocirc);
+      } else if ((reason & ~END_CIRC_REASON_FLAG_REMOTE)
+                  == END_CIRC_REASON_CHANNEL_CLOSED &&
+                 circ->n_chan &&
+                 circ->n_chan->reason_for_closing
+                  != CHANNEL_CLOSE_REQUESTED) {
+        /* If we didn't close the channel ourselves, it could be bias */
+        /* XXX: Only count bias if the network is live?
+         * What about clock jumps/suspends? */
+        log_info(LD_CIRC,
+            "Circuit %d's channel closed without successful use for reason "
+            "%d, channel reason %d. Circuit purpose %d currently %d,%s. Len "
+            "%d.", ocirc->global_identifier,
+            reason, circ->n_chan->reason_for_closing,
+            circ->purpose, ocirc->has_opened,
+            circuit_state_to_string(circ->state),
+            ocirc->build_state->desired_path_len);
+        pathbias_count_collapse(ocirc);
+      } else {
+        pathbias_count_successful_close(ocirc);
+      }
+      break;
+
+    /* If we tried to use a circuit but failed, we should probe it to ensure
+     * it has not been tampered with. */
+    case PATH_STATE_USE_ATTEMPTED:
+      /* XXX: Only probe and/or count failure if the network is live?
+       * What about clock jumps/suspends? */
+      if (pathbias_send_usable_probe(circ) == 0)
+        return -1;
+      else
+        pathbias_count_use_failed(ocirc);
+
+      /* Any circuit where there were attempted streams but no successful
+       * streams could be bias */
+      log_info(LD_CIRC,
+            "Circuit %d closed without successful use for reason %d. "
+            "Circuit purpose %d currently %d,%s. Len %d.",
+            ocirc->global_identifier,
+            reason, circ->purpose, ocirc->has_opened,
+            circuit_state_to_string(circ->state),
+            ocirc->build_state->desired_path_len);
+      break;
+
+    case PATH_STATE_USE_SUCCEEDED:
+      pathbias_count_successful_close(ocirc);
+      pathbias_count_use_success(ocirc);
+      break;
+
+    case PATH_STATE_USE_FAILED:
+      pathbias_count_use_failed(ocirc);
+      break;
+
+    case PATH_STATE_NEW_CIRC:
+    case PATH_STATE_BUILD_ATTEMPTED:
+    case PATH_STATE_ALREADY_COUNTED:
+    default:
+      // Other states are uninteresting. No stats to count.
+      break;
+  }
+
+  ocirc->path_state = PATH_STATE_ALREADY_COUNTED;
+
+  return 0;
+}
+
+/**
+ * Count a successfully closed circuit.
+ */
+static void
+pathbias_count_successful_close(origin_circuit_t *circ)
+{
+  entry_guard_t *guard = NULL;
+  if (!pathbias_should_count(circ)) {
+    return;
+  }
+
+  if (circ->cpath && circ->cpath->extend_info) {
+    guard = entry_guard_get_by_id_digest(
+              circ->cpath->extend_info->identity_digest);
+  }
+
+  if (guard) {
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    /* In the long run: circuit_success ~= successful_circuit_close +
+     *                                     circ_failure + stream_failure */
+    pb->successful_circuits_closed++;
+    entry_guards_changed();
+  } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+   /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
+    * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here.
+    * No need to log that case. */
+    log_info(LD_CIRC,
+        "Successfully closed circuit has no known guard. "
+        "Circuit is a %s currently %s",
+        circuit_purpose_to_string(circ->base_.purpose),
+        circuit_state_to_string(circ->base_.state));
+  }
+}
+
+/**
+ * Count a circuit that fails after it is built, but before it can
+ * carry any traffic.
+ *
+ * This is needed because there are ways to destroy a
+ * circuit after it has successfully completed. Right now, this is
+ * used for purely informational/debugging purposes.
+ */
+static void
+pathbias_count_collapse(origin_circuit_t *circ)
+{
+  entry_guard_t *guard = NULL;
+
+  if (!pathbias_should_count(circ)) {
+    return;
+  }
+
+  if (circ->cpath && circ->cpath->extend_info) {
+    guard = entry_guard_get_by_id_digest(
+              circ->cpath->extend_info->identity_digest);
+  }
+
+  if (guard) {
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->collapsed_circuits++;
+    entry_guards_changed();
+  } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+   /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
+    * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here.
+    * No need to log that case. */
+    log_info(LD_CIRC,
+        "Destroyed circuit has no known guard. "
+        "Circuit is a %s currently %s",
+        circuit_purpose_to_string(circ->base_.purpose),
+        circuit_state_to_string(circ->base_.state));
+  }
+}
+
+/**
+ * Count a known failed circuit (because we could not probe it).
+ *
+ * This counter is informational.
+ */
+static void
+pathbias_count_use_failed(origin_circuit_t *circ)
+{
+  entry_guard_t *guard = NULL;
+  if (!pathbias_should_count(circ)) {
+    return;
+  }
+
+  if (circ->cpath && circ->cpath->extend_info) {
+    guard = entry_guard_get_by_id_digest(
+              circ->cpath->extend_info->identity_digest);
+  }
+
+  if (guard) {
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->unusable_circuits++;
+    entry_guards_changed();
+  } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+   /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
+    * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here.
+    * No need to log that case. */
+    /* XXX note cut-and-paste code in this function compared to nearby
+     * functions. Would be nice to refactor. -RD */
+    log_info(LD_CIRC,
+        "Stream-failing circuit has no known guard. "
+        "Circuit is a %s currently %s",
+        circuit_purpose_to_string(circ->base_.purpose),
+        circuit_state_to_string(circ->base_.state));
+  }
+}
+
+/**
+ * Count timeouts for path bias log messages.
+ *
+ * These counts are purely informational.
+ */
+void
+pathbias_count_timeout(origin_circuit_t *circ)
+{
+  entry_guard_t *guard = NULL;
+
+  if (!pathbias_should_count(circ)) {
+    return;
+  }
+
+  /* For hidden service circs, they can actually be used
+   * successfully and then time out later (because
+   * the other side declines to use them). */
+  if (circ->path_state == PATH_STATE_USE_SUCCEEDED) {
+    return;
+  }
+
+  if (circ->cpath && circ->cpath->extend_info) {
+    guard = entry_guard_get_by_id_digest(
+              circ->cpath->extend_info->identity_digest);
+  }
+
+  if (guard) {
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->timeouts++;
+    entry_guards_changed();
+  }
+}
+
+/**
+ * Helper function to count all of the currently opened circuits
+ * for a guard that are in a given path state range. The state
+ * range is inclusive on both ends.
+ */
+static int
+pathbias_count_circs_in_states(entry_guard_t *guard,
+                              path_state_t from,
+                              path_state_t to)
+{
+  int open_circuits = 0;
+
+  /* Count currently open circuits. Give them the benefit of the doubt. */
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    origin_circuit_t *ocirc = NULL;
+    if (!CIRCUIT_IS_ORIGIN(circ) || /* didn't originate here */
+        circ->marked_for_close) /* already counted */
+      continue;
+
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+    if (!ocirc->cpath || !ocirc->cpath->extend_info)
+      continue;
+
+    if (ocirc->path_state >= from &&
+        ocirc->path_state <= to &&
+        pathbias_should_count(ocirc) &&
+        fast_memeq(entry_guard_get_rsa_id_digest(guard),
+                   ocirc->cpath->extend_info->identity_digest,
+                   DIGEST_LEN)) {
+      log_debug(LD_CIRC, "Found opened circuit %d in path_state %s",
+                ocirc->global_identifier,
+                pathbias_state_to_string(ocirc->path_state));
+      open_circuits++;
+    }
+  }
+  SMARTLIST_FOREACH_END(circ);
+
+  return open_circuits;
+}
+
+/**
+ * Return the number of circuits counted as successfully closed for
+ * this guard.
+ *
+ * Also add in the currently open circuits to give them the benefit
+ * of the doubt.
+ */
+double
+pathbias_get_close_success_count(entry_guard_t *guard)
+{
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  return pb->successful_circuits_closed +
+         pathbias_count_circs_in_states(guard,
+                       PATH_STATE_BUILD_SUCCEEDED,
+                       PATH_STATE_USE_SUCCEEDED);
+}
+
+/**
+ * Return the number of circuits counted as successfully used
+ * this guard.
+ *
+ * Also add in the currently open circuits that we are attempting
+ * to use to give them the benefit of the doubt.
+ */
+double
+pathbias_get_use_success_count(entry_guard_t *guard)
+{
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  return pb->use_successes +
+         pathbias_count_circs_in_states(guard,
+                       PATH_STATE_USE_ATTEMPTED,
+                       PATH_STATE_USE_SUCCEEDED);
+}
+
+/**
+ * Check the path bias use rate against our consensus parameter limits.
+ *
+ * Emits a log message if the use success rates are too low.
+ *
+ * If pathbias_get_dropguards() is set, we also disable the use of
+ * very failure prone guards.
+ */
+static void
+pathbias_measure_use_rate(entry_guard_t *guard)
+{
+  const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  if (pb->use_attempts > pathbias_get_min_use(options)) {
+    /* Note: We rely on the < comparison here to allow us to set a 0
+     * rate and disable the feature entirely. If refactoring, don't
+     * change to <= */
+    if (pathbias_get_use_success_count(guard)/pb->use_attempts
+        < pathbias_get_extreme_use_rate(options)) {
+      /* Dropping is currently disabled by default. */
+      if (pathbias_get_dropguards(options)) {
+        if (!pb->path_bias_disabled) {
+          log_warn(LD_CIRC,
+                 "Your Guard %s is failing to carry an extremely large "
+                 "amount of stream on its circuits. "
+                 "To avoid potential route manipulation attacks, Tor has "
+                 "disabled use of this guard. "
+                 "Use counts are %ld/%ld. Success counts are %ld/%ld. "
+                 "%ld circuits completed, %ld were unusable, %ld collapsed, "
+                 "and %ld timed out. "
+                 "For reference, your timeout cutoff is %ld seconds.",
+                 entry_guard_describe(guard),
+                 tor_lround(pathbias_get_use_success_count(guard)),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pathbias_get_close_success_count(guard)),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
+                 tor_lround(get_circuit_build_close_time_ms()/1000));
+          pb->path_bias_disabled = 1;
+          return;
+        }
+      } else if (!pb->path_bias_use_extreme) {
+        pb->path_bias_use_extreme = 1;
+        log_warn(LD_CIRC,
+                 "Your Guard %s is failing to carry an extremely large "
+                 "amount of streams on its circuits. "
+                 "This could indicate a route manipulation attack, network "
+                 "overload, bad local network connectivity, or a bug. "
+                 "Use counts are %ld/%ld. Success counts are %ld/%ld. "
+                 "%ld circuits completed, %ld were unusable, %ld collapsed, "
+                 "and %ld timed out. "
+                 "For reference, your timeout cutoff is %ld seconds.",
+                 entry_guard_describe(guard),
+                 tor_lround(pathbias_get_use_success_count(guard)),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pathbias_get_close_success_count(guard)),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
+                 tor_lround(get_circuit_build_close_time_ms()/1000));
+      }
+    } else if (pathbias_get_use_success_count(guard)/pb->use_attempts
+               < pathbias_get_notice_use_rate(options)) {
+      if (!pb->path_bias_use_noticed) {
+        pb->path_bias_use_noticed = 1;
+        log_notice(LD_CIRC,
+                 "Your Guard %s is failing to carry more streams on its "
+                 "circuits than usual. "
+                 "Most likely this means the Tor network is overloaded "
+                 "or your network connection is poor. "
+                 "Use counts are %ld/%ld. Success counts are %ld/%ld. "
+                 "%ld circuits completed, %ld were unusable, %ld collapsed, "
+                 "and %ld timed out. "
+                 "For reference, your timeout cutoff is %ld seconds.",
+                 entry_guard_describe(guard),
+                 tor_lround(pathbias_get_use_success_count(guard)),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pathbias_get_close_success_count(guard)),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
+                 tor_lround(get_circuit_build_close_time_ms()/1000));
+      }
+    }
+  }
+}
+
+/**
+ * Check the path bias circuit close status rates against our consensus
+ * parameter limits.
+ *
+ * Emits a log message if the use success rates are too low.
+ *
+ * If pathbias_get_dropguards() is set, we also disable the use of
+ * very failure prone guards.
+ *
+ * XXX: This function shares similar log messages and checks to
+ * pathbias_measure_use_rate(). It may be possible to combine them
+ * eventually, especially if we can ever remove the need for 3
+ * levels of closure warns (if the overall circuit failure rate
+ * goes down with ntor). One way to do so would be to multiply
+ * the build rate with the use rate to get an idea of the total
+ * fraction of the total network paths the user is able to use.
+ * See ticket #8159.
+ */
+static void
+pathbias_measure_close_rate(entry_guard_t *guard)
+{
+  const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  if (pb->circ_attempts > pathbias_get_min_circs(options)) {
+    /* Note: We rely on the < comparison here to allow us to set a 0
+     * rate and disable the feature entirely. If refactoring, don't
+     * change to <= */
+    if (pathbias_get_close_success_count(guard)/pb->circ_attempts
+        < pathbias_get_extreme_rate(options)) {
+      /* Dropping is currently disabled by default. */
+      if (pathbias_get_dropguards(options)) {
+        if (!pb->path_bias_disabled) {
+          log_warn(LD_CIRC,
+                 "Your Guard %s is failing an extremely large "
+                 "amount of circuits. "
+                 "To avoid potential route manipulation attacks, Tor has "
+                 "disabled use of this guard. "
+                 "Success counts are %ld/%ld. Use counts are %ld/%ld. "
+                 "%ld circuits completed, %ld were unusable, %ld collapsed, "
+                 "and %ld timed out. "
+                 "For reference, your timeout cutoff is %ld seconds.",
+                 entry_guard_describe(guard),
+                 tor_lround(pathbias_get_close_success_count(guard)),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pathbias_get_use_success_count(guard)),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
+                 tor_lround(get_circuit_build_close_time_ms()/1000));
+          pb->path_bias_disabled = 1;
+          return;
+        }
+      } else if (!pb->path_bias_extreme) {
+        pb->path_bias_extreme = 1;
+        log_warn(LD_CIRC,
+                 "Your Guard %s is failing an extremely large "
+                 "amount of circuits. "
+                 "This could indicate a route manipulation attack, "
+                 "extreme network overload, or a bug. "
+                 "Success counts are %ld/%ld. Use counts are %ld/%ld. "
+                 "%ld circuits completed, %ld were unusable, %ld collapsed, "
+                 "and %ld timed out. "
+                 "For reference, your timeout cutoff is %ld seconds.",
+                 entry_guard_describe(guard),
+                 tor_lround(pathbias_get_close_success_count(guard)),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pathbias_get_use_success_count(guard)),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
+                 tor_lround(get_circuit_build_close_time_ms()/1000));
+      }
+    } else if (pathbias_get_close_success_count(guard)/pb->circ_attempts
+                < pathbias_get_warn_rate(options)) {
+      if (!pb->path_bias_warned) {
+        pb->path_bias_warned = 1;
+        log_warn(LD_CIRC,
+                 "Your Guard %s is failing a very large "
+                 "amount of circuits. "
+                 "Most likely this means the Tor network is "
+                 "overloaded, but it could also mean an attack against "
+                 "you or potentially the guard itself. "
+                 "Success counts are %ld/%ld. Use counts are %ld/%ld. "
+                 "%ld circuits completed, %ld were unusable, %ld collapsed, "
+                 "and %ld timed out. "
+                 "For reference, your timeout cutoff is %ld seconds.",
+                 entry_guard_describe(guard),
+                 tor_lround(pathbias_get_close_success_count(guard)),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pathbias_get_use_success_count(guard)),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
+                 tor_lround(get_circuit_build_close_time_ms()/1000));
+      }
+    } else if (pathbias_get_close_success_count(guard)/pb->circ_attempts
+               < pathbias_get_notice_rate(options)) {
+      if (!pb->path_bias_noticed) {
+        pb->path_bias_noticed = 1;
+        log_notice(LD_CIRC,
+                 "Your Guard %s is failing more circuits than "
+                 "usual. "
+                 "Most likely this means the Tor network is overloaded. "
+                 "Success counts are %ld/%ld. Use counts are %ld/%ld. "
+                 "%ld circuits completed, %ld were unusable, %ld collapsed, "
+                 "and %ld timed out. "
+                 "For reference, your timeout cutoff is %ld seconds.",
+                 entry_guard_describe(guard),
+                 tor_lround(pathbias_get_close_success_count(guard)),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pathbias_get_use_success_count(guard)),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
+                 tor_lround(get_circuit_build_close_time_ms()/1000));
+      }
+    }
+  }
+}
+
+/**
+ * This function scales the path bias use rates if we have
+ * more data than the scaling threshold. This allows us to
+ * be more sensitive to recent measurements.
+ *
+ * XXX: The attempt count transfer stuff here might be done
+ * better by keeping separate pending counters that get
+ * transferred at circuit close. See ticket #8160.
+ */
+static void
+pathbias_scale_close_rates(entry_guard_t *guard)
+{
+  const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  /* If we get a ton of circuits, just scale everything down */
+  if (pb->circ_attempts > pathbias_get_scale_threshold(options)) {
+    double scale_ratio = pathbias_get_scale_ratio(options);
+    int opened_attempts = pathbias_count_circs_in_states(guard,
+            PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED);
+    int opened_built = pathbias_count_circs_in_states(guard,
+                        PATH_STATE_BUILD_SUCCEEDED,
+                        PATH_STATE_USE_FAILED);
+    /* Verify that the counts are sane before and after scaling */
+    int counts_are_sane = (pb->circ_attempts >= pb->circ_successes);
+
+    pb->circ_attempts -= (opened_attempts+opened_built);
+    pb->circ_successes -= opened_built;
+
+    pb->circ_attempts *= scale_ratio;
+    pb->circ_successes *= scale_ratio;
+    pb->timeouts *= scale_ratio;
+    pb->successful_circuits_closed *= scale_ratio;
+    pb->collapsed_circuits *= scale_ratio;
+    pb->unusable_circuits *= scale_ratio;
+
+    pb->circ_attempts += (opened_attempts+opened_built);
+    pb->circ_successes += opened_built;
+
+    entry_guards_changed();
+
+    log_info(LD_CIRC,
+             "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard "
+             "%s",
+             pb->circ_successes, pb->successful_circuits_closed,
+             pb->circ_attempts, opened_built, opened_attempts,
+             entry_guard_describe(guard));
+
+    /* Have the counts just become invalid by this scaling attempt? */
+    if (counts_are_sane && pb->circ_attempts < pb->circ_successes) {
+      log_notice(LD_BUG,
+               "Scaling has mangled pathbias counts to %f/%f (%d/%d open) "
+               "for guard %s",
+               pb->circ_successes, pb->circ_attempts, opened_built,
+               opened_attempts,
+               entry_guard_describe(guard));
+    }
+  }
+}
+
+/**
+ * This function scales the path bias circuit close rates if we have
+ * more data than the scaling threshold. This allows us to be more
+ * sensitive to recent measurements.
+ *
+ * XXX: The attempt count transfer stuff here might be done
+ * better by keeping separate pending counters that get
+ * transferred at circuit close. See ticket #8160.
+ */
+void
+pathbias_scale_use_rates(entry_guard_t *guard)
+{
+  const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  /* If we get a ton of circuits, just scale everything down */
+  if (pb->use_attempts > pathbias_get_scale_use_threshold(options)) {
+    double scale_ratio = pathbias_get_scale_ratio(options);
+    int opened_attempts = pathbias_count_circs_in_states(guard,
+            PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED);
+    /* Verify that the counts are sane before and after scaling */
+    int counts_are_sane = (pb->use_attempts >= pb->use_successes);
+
+    pb->use_attempts -= opened_attempts;
+
+    pb->use_attempts *= scale_ratio;
+    pb->use_successes *= scale_ratio;
+
+    pb->use_attempts += opened_attempts;
+
+    log_info(LD_CIRC,
+           "Scaled pathbias use counts to %f/%f (%d open) for guard %s",
+           pb->use_successes, pb->use_attempts, opened_attempts,
+           entry_guard_describe(guard));
+
+    /* Have the counts just become invalid by this scaling attempt? */
+    if (counts_are_sane && pb->use_attempts < pb->use_successes) {
+      log_notice(LD_BUG,
+               "Scaling has mangled pathbias usage counts to %f/%f "
+               "(%d open) for guard %s",
+               pb->circ_successes, pb->circ_attempts,
+               opened_attempts, entry_guard_describe(guard));
+    }
+
+    entry_guards_changed();
+  }
+}
diff --git a/src/feature/client/circpathbias.h b/src/feature/client/circpathbias.h
new file mode 100644
index 0000000000..c99d1277bb
--- /dev/null
+++ b/src/feature/client/circpathbias.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitbuild.h
+ * \brief Header file for circuitbuild.c.
+ **/
+
+#ifndef TOR_CIRCPATHBIAS_H
+#define TOR_CIRCPATHBIAS_H
+
+double pathbias_get_extreme_rate(const or_options_t *options);
+double pathbias_get_extreme_use_rate(const or_options_t *options);
+int pathbias_get_dropguards(const or_options_t *options);
+void pathbias_count_timeout(origin_circuit_t *circ);
+void pathbias_count_build_success(origin_circuit_t *circ);
+int pathbias_count_build_attempt(origin_circuit_t *circ);
+int pathbias_check_close(origin_circuit_t *circ, int reason);
+int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell);
+void pathbias_count_use_attempt(origin_circuit_t *circ);
+void pathbias_mark_use_success(origin_circuit_t *circ);
+void pathbias_mark_use_rollback(origin_circuit_t *circ);
+const char *pathbias_state_to_string(enum path_state_t state);
+
+#endif /* !defined(TOR_CIRCPATHBIAS_H) */
diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c
new file mode 100644
index 0000000000..6e75254239
--- /dev/null
+++ b/src/feature/client/dnsserv.c
@@ -0,0 +1,415 @@
+/* Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dnsserv.c
+ * \brief Implements client-side DNS proxy server code.
+ *
+ * When a user enables the DNSPort configuration option to have their local
+ * Tor client handle DNS requests, this module handles it.  It functions as a
+ * "DNS Server" on the client side, which client applications use.
+ *
+ * Inbound DNS requests are represented as entry_connection_t here (since
+ * that's how Tor represents client-side streams), which are kept associated
+ * with an evdns_server_request structure as exposed by Libevent's
+ * evdns code.
+ *
+ * Upon receiving a DNS request, libevent calls our evdns_server_callback()
+ * function here, which causes this module to create an entry_connection_t
+ * request as appropriate.  Later, when that request is answered,
+ * connection_edge.c calls dnsserv_resolved() so we can finish up and tell the
+ * DNS client.
+ **/
+
+#include "or/or.h"
+#include "or/dnsserv.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/connection_edge.h"
+#include "or/control.h"
+#include "or/main.h"
+#include "or/policies.h"
+
+#include "or/control_connection_st.h"
+#include "or/entry_connection_st.h"
+#include "or/listener_connection_st.h"
+#include "or/socks_request_st.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include <event2/dns.h>
+#include <event2/dns_compat.h>
+/* XXXX this implies we want an improved evdns  */
+#include <event2/dns_struct.h>
+
+/** Helper function: called by evdns whenever the client sends a request to our
+ * DNSPort.  We need to eventually answer the request <b>req</b>.
+ */
+static void
+evdns_server_callback(struct evdns_server_request *req, void *data_)
+{
+  const listener_connection_t *listener = data_;
+  entry_connection_t *entry_conn;
+  edge_connection_t *conn;
+  int i = 0;
+  struct evdns_server_question *q = NULL, *supported_q = NULL;
+  struct sockaddr_storage addr;
+  struct sockaddr *sa;
+  int addrlen;
+  tor_addr_t tor_addr;
+  uint16_t port;
+  int err = DNS_ERR_NONE;
+  char *q_name;
+
+  tor_assert(req);
+
+  log_info(LD_APP, "Got a new DNS request!");
+
+  req->flags |= 0x80; /* set RA */
+
+  /* First, check whether the requesting address matches our SOCKSPolicy. */
+  if ((addrlen = evdns_server_request_get_requesting_addr(req,
+                      (struct sockaddr*)&addr, (socklen_t)sizeof(addr))) < 0) {
+    log_warn(LD_APP, "Couldn't get requesting address.");
+    evdns_server_request_respond(req, DNS_ERR_SERVERFAILED);
+    return;
+  }
+  (void) addrlen;
+  sa = (struct sockaddr*) &addr;
+  if (tor_addr_from_sockaddr(&tor_addr, sa, &port)<0) {
+    log_warn(LD_APP, "Requesting address wasn't recognized.");
+    evdns_server_request_respond(req, DNS_ERR_SERVERFAILED);
+    return;
+  }
+
+  if (!socks_policy_permits_address(&tor_addr)) {
+    log_warn(LD_APP, "Rejecting DNS request from disallowed IP.");
+    evdns_server_request_respond(req, DNS_ERR_REFUSED);
+    return;
+  }
+
+  /* Now, let's find the first actual question of a type we can answer in this
+   * DNS request.  It makes us a little noncompliant to act like this; we
+   * should fix that eventually if it turns out to make a difference for
+   * anybody. */
+  if (req->nquestions == 0) {
+    log_info(LD_APP, "No questions in DNS request; sending back nil reply.");
+    evdns_server_request_respond(req, 0);
+    return;
+  }
+  if (req->nquestions > 1) {
+    log_info(LD_APP, "Got a DNS request with more than one question; I only "
+             "handle one question at a time for now.  Skipping the extras.");
+  }
+  for (i = 0; i < req->nquestions; ++i) {
+    if (req->questions[i]->dns_question_class != EVDNS_CLASS_INET)
+      continue;
+    switch (req->questions[i]->type) {
+      case EVDNS_TYPE_A:
+      case EVDNS_TYPE_AAAA:
+      case EVDNS_TYPE_PTR:
+        /* We always pick the first one of these questions, if there is
+           one. */
+        if (! supported_q)
+          supported_q = req->questions[i];
+        break;
+      default:
+        break;
+      }
+  }
+  if (supported_q)
+    q = supported_q;
+  if (!q) {
+    log_info(LD_APP, "None of the questions we got were ones we're willing "
+             "to support. Sending NOTIMPL.");
+    evdns_server_request_respond(req, DNS_ERR_NOTIMPL);
+    return;
+  }
+
+  /* Make sure the name isn't too long: This should be impossible, I think. */
+  if (err == DNS_ERR_NONE && strlen(q->name) > MAX_SOCKS_ADDR_LEN-1)
+    err = DNS_ERR_FORMAT;
+
+  if (err != DNS_ERR_NONE || !supported_q) {
+    /* We got an error?  There's no question we're willing to answer? Then
+     * send back an answer immediately; we're done. */
+    evdns_server_request_respond(req, err);
+    return;
+  }
+
+  /* Make a new dummy AP connection, and attach the request to it. */
+  entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET);
+  conn = ENTRY_TO_EDGE_CONN(entry_conn);
+  CONNECTION_AP_EXPECT_NONPENDING(entry_conn);
+  TO_CONN(conn)->state = AP_CONN_STATE_RESOLVE_WAIT;
+  conn->is_dns_request = 1;
+
+  tor_addr_copy(&TO_CONN(conn)->addr, &tor_addr);
+  TO_CONN(conn)->port = port;
+  TO_CONN(conn)->address = tor_addr_to_str_dup(&tor_addr);
+
+  if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA ||
+      q->type == EVDNS_QTYPE_ALL) {
+    entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE;
+  } else {
+    tor_assert(q->type == EVDNS_TYPE_PTR);
+    entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE_PTR;
+  }
+
+  /* This serves our DNS port so enable DNS request by default. */
+  entry_conn->entry_cfg.dns_request = 1;
+  if (q->type == EVDNS_TYPE_A || q->type == EVDNS_QTYPE_ALL) {
+    entry_conn->entry_cfg.ipv4_traffic = 1;
+    entry_conn->entry_cfg.ipv6_traffic = 0;
+    entry_conn->entry_cfg.prefer_ipv6 = 0;
+  } else if (q->type == EVDNS_TYPE_AAAA) {
+    entry_conn->entry_cfg.ipv4_traffic = 0;
+    entry_conn->entry_cfg.ipv6_traffic = 1;
+    entry_conn->entry_cfg.prefer_ipv6 = 1;
+  }
+
+  strlcpy(entry_conn->socks_request->address, q->name,
+          sizeof(entry_conn->socks_request->address));
+
+  entry_conn->socks_request->listener_type = listener->base_.type;
+  entry_conn->dns_server_request = req;
+  entry_conn->entry_cfg.isolation_flags = listener->entry_cfg.isolation_flags;
+  entry_conn->entry_cfg.session_group = listener->entry_cfg.session_group;
+  entry_conn->nym_epoch = get_signewnym_epoch();
+
+  if (connection_add(ENTRY_TO_CONN(entry_conn)) < 0) {
+    log_warn(LD_APP, "Couldn't register dummy connection for DNS request");
+    evdns_server_request_respond(req, DNS_ERR_SERVERFAILED);
+    connection_free_(ENTRY_TO_CONN(entry_conn));
+    return;
+  }
+
+  control_event_stream_status(entry_conn, STREAM_EVENT_NEW_RESOLVE, 0);
+
+  /* Now, unless a controller asked us to leave streams unattached,
+  * throw the connection over to get rewritten (which will
+  * answer it immediately if it's in the cache, or completely bogus, or
+  * automapped), and then attached to a circuit. */
+  log_info(LD_APP, "Passing request for %s to rewrite_and_attach.",
+           escaped_safe_str_client(q->name));
+  q_name = tor_strdup(q->name); /* q could be freed in rewrite_and_attach */
+  connection_ap_rewrite_and_attach_if_allowed(entry_conn, NULL, NULL);
+  /* Now, the connection is marked if it was bad. */
+
+  log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.",
+           escaped_safe_str_client(q_name));
+  tor_free(q_name);
+}
+
+/** Helper function: called whenever the client sends a resolve request to our
+ * controller.  We need to eventually answer the request <b>req</b>.
+ * Returns 0 if the controller will be getting (or has gotten) an event in
+ * response; -1 if we couldn't launch the request.
+ */
+int
+dnsserv_launch_request(const char *name, int reverse,
+                       control_connection_t *control_conn)
+{
+  entry_connection_t *entry_conn;
+  edge_connection_t *conn;
+  char *q_name;
+
+  /* Make a new dummy AP connection, and attach the request to it. */
+  entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET);
+  entry_conn->entry_cfg.dns_request = 1;
+  conn = ENTRY_TO_EDGE_CONN(entry_conn);
+  CONNECTION_AP_EXPECT_NONPENDING(entry_conn);
+  conn->base_.state = AP_CONN_STATE_RESOLVE_WAIT;
+
+  tor_addr_copy(&TO_CONN(conn)->addr, &control_conn->base_.addr);
+#ifdef AF_UNIX
+  /*
+   * The control connection can be AF_UNIX and if so tor_addr_to_str_dup will
+   * unhelpfully say "<unknown address type>"; say "(Tor_internal)"
+   * instead.
+   */
+  if (control_conn->base_.socket_family == AF_UNIX) {
+    TO_CONN(conn)->port = 0;
+    TO_CONN(conn)->address = tor_strdup("(Tor_internal)");
+  } else {
+    TO_CONN(conn)->port = control_conn->base_.port;
+    TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
+  }
+#else /* !(defined(AF_UNIX)) */
+  TO_CONN(conn)->port = control_conn->base_.port;
+  TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
+#endif /* defined(AF_UNIX) */
+
+  if (reverse)
+    entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE_PTR;
+  else
+    entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE;
+
+  conn->is_dns_request = 1;
+
+  strlcpy(entry_conn->socks_request->address, name,
+          sizeof(entry_conn->socks_request->address));
+
+  entry_conn->socks_request->listener_type = CONN_TYPE_CONTROL_LISTENER;
+  entry_conn->original_dest_address = tor_strdup(name);
+  entry_conn->entry_cfg.session_group = SESSION_GROUP_CONTROL_RESOLVE;
+  entry_conn->nym_epoch = get_signewnym_epoch();
+  entry_conn->entry_cfg.isolation_flags = ISO_DEFAULT;
+
+  if (connection_add(TO_CONN(conn))<0) {
+    log_warn(LD_APP, "Couldn't register dummy connection for RESOLVE request");
+    connection_free_(TO_CONN(conn));
+    return -1;
+  }
+
+  control_event_stream_status(entry_conn, STREAM_EVENT_NEW_RESOLVE, 0);
+
+  /* Now, unless a controller asked us to leave streams unattached,
+  * throw the connection over to get rewritten (which will
+  * answer it immediately if it's in the cache, or completely bogus, or
+  * automapped), and then attached to a circuit. */
+  log_info(LD_APP, "Passing request for %s to rewrite_and_attach.",
+           escaped_safe_str_client(name));
+  q_name = tor_strdup(name); /* q could be freed in rewrite_and_attach */
+  connection_ap_rewrite_and_attach_if_allowed(entry_conn, NULL, NULL);
+  /* Now, the connection is marked if it was bad. */
+
+  log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.",
+           escaped_safe_str_client(q_name));
+  tor_free(q_name);
+  return 0;
+}
+
+/** If there is a pending request on <b>conn</b> that's waiting for an answer,
+ * send back an error and free the request. */
+void
+dnsserv_reject_request(entry_connection_t *conn)
+{
+  if (conn->dns_server_request) {
+    evdns_server_request_respond(conn->dns_server_request,
+                                 DNS_ERR_SERVERFAILED);
+    conn->dns_server_request = NULL;
+  }
+}
+
+/** Look up the original name that corresponds to 'addr' in req.  We use this
+ * to preserve case in order to facilitate clients using 0x20-hacks to avoid
+ * DNS poisoning. */
+static const char *
+evdns_get_orig_address(const struct evdns_server_request *req,
+                       int rtype, const char *addr)
+{
+  int i, type;
+
+  switch (rtype) {
+  case RESOLVED_TYPE_IPV4:
+    type = EVDNS_TYPE_A;
+    break;
+  case RESOLVED_TYPE_HOSTNAME:
+    type = EVDNS_TYPE_PTR;
+    break;
+  case RESOLVED_TYPE_IPV6:
+    type = EVDNS_TYPE_AAAA;
+    break;
+  case RESOLVED_TYPE_ERROR:
+  case RESOLVED_TYPE_ERROR_TRANSIENT:
+     /* Addr doesn't matter, since we're not sending it back in the reply.*/
+    return addr;
+  default:
+    tor_fragile_assert();
+    return addr;
+  }
+
+  for (i = 0; i < req->nquestions; ++i) {
+    const struct evdns_server_question *q = req->questions[i];
+    if (q->type == type && !strcasecmp(q->name, addr))
+      return q->name;
+  }
+  return addr;
+}
+
+/** Tell the dns request waiting for an answer on <b>conn</b> that we have an
+ * answer of type <b>answer_type</b> (RESOLVE_TYPE_IPV4/IPV6/ERR), of length
+ * <b>answer_len</b>, in <b>answer</b>, with TTL <b>ttl</b>.  Doesn't do
+ * any caching; that's handled elsewhere. */
+void
+dnsserv_resolved(entry_connection_t *conn,
+                 int answer_type,
+                 size_t answer_len,
+                 const char *answer,
+                 int ttl)
+{
+  struct evdns_server_request *req = conn->dns_server_request;
+  const char *name;
+  int err = DNS_ERR_NONE;
+  if (!req)
+    return;
+  name = evdns_get_orig_address(req, answer_type,
+                                conn->socks_request->address);
+
+  /* XXXX Re-do; this is dumb. */
+  if (ttl < 60)
+    ttl = 60;
+
+  /* The evdns interface is: add a bunch of reply items (corresponding to one
+   * or more of the questions in the request); then, call
+   * evdns_server_request_respond. */
+  if (answer_type == RESOLVED_TYPE_IPV6) {
+    evdns_server_request_add_aaaa_reply(req,
+                                        name,
+                                        1, answer, ttl);
+  } else if (answer_type == RESOLVED_TYPE_IPV4 && answer_len == 4 &&
+             conn->socks_request->command == SOCKS_COMMAND_RESOLVE) {
+    evdns_server_request_add_a_reply(req,
+                                     name,
+                                     1, answer, ttl);
+  } else if (answer_type == RESOLVED_TYPE_HOSTNAME &&
+             answer_len < 256 &&
+             conn->socks_request->command == SOCKS_COMMAND_RESOLVE_PTR) {
+    char *ans = tor_strndup(answer, answer_len);
+    evdns_server_request_add_ptr_reply(req, NULL,
+                                       name,
+                                       ans, ttl);
+    tor_free(ans);
+  } else if (answer_type == RESOLVED_TYPE_ERROR) {
+    err = DNS_ERR_NOTEXIST;
+  } else { /* answer_type == RESOLVED_TYPE_ERROR_TRANSIENT */
+    err = DNS_ERR_SERVERFAILED;
+  }
+
+  evdns_server_request_respond(req, err);
+
+  conn->dns_server_request = NULL;
+}
+
+/** Set up the evdns server port for the UDP socket on <b>conn</b>, which
+ * must be an AP_DNS_LISTENER */
+void
+dnsserv_configure_listener(connection_t *conn)
+{
+  listener_connection_t *listener_conn;
+  tor_assert(conn);
+  tor_assert(SOCKET_OK(conn->s));
+  tor_assert(conn->type == CONN_TYPE_AP_DNS_LISTENER);
+
+  listener_conn = TO_LISTENER_CONN(conn);
+  listener_conn->dns_server_port =
+    tor_evdns_add_server_port(conn->s, 0, evdns_server_callback,
+                              listener_conn);
+}
+
+/** Free the evdns server port for <b>conn</b>, which must be an
+ * AP_DNS_LISTENER. */
+void
+dnsserv_close_listener(connection_t *conn)
+{
+  listener_connection_t *listener_conn;
+  tor_assert(conn);
+  tor_assert(conn->type == CONN_TYPE_AP_DNS_LISTENER);
+
+  listener_conn = TO_LISTENER_CONN(conn);
+
+  if (listener_conn->dns_server_port) {
+    evdns_close_server_port(listener_conn->dns_server_port);
+    listener_conn->dns_server_port = NULL;
+  }
+}
diff --git a/src/feature/client/dnsserv.h b/src/feature/client/dnsserv.h
new file mode 100644
index 0000000000..afdde3a342
--- /dev/null
+++ b/src/feature/client/dnsserv.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dnsserv.h
+ * \brief Header file for dnsserv.c.
+ **/
+
+#ifndef TOR_DNSSERV_H
+#define TOR_DNSSERV_H
+
+void dnsserv_configure_listener(connection_t *conn);
+void dnsserv_close_listener(connection_t *conn);
+void dnsserv_resolved(entry_connection_t *conn,
+                      int answer_type,
+                      size_t answer_len,
+                      const char *answer,
+                      int ttl);
+void dnsserv_reject_request(entry_connection_t *conn);
+int dnsserv_launch_request(const char *name, int is_reverse,
+                           control_connection_t *control_conn);
+
+#endif /* !defined(TOR_DNSSERV_H) */
+
diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c
new file mode 100644
index 0000000000..ba9c30f8b3
--- /dev/null
+++ b/src/feature/client/entrynodes.c
@@ -0,0 +1,3694 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file entrynodes.c
+ * \brief Code to manage our fixed first nodes for various functions.
+ *
+ * Entry nodes can be guards (for general use) or bridges (for censorship
+ * circumvention).
+ *
+ * In general, we use entry guards to prevent traffic-sampling attacks:
+ * if we chose every circuit independently, an adversary controlling
+ * some fraction of paths on the network would observe a sample of every
+ * user's traffic. Using guards gives users a chance of not being
+ * profiled.
+ *
+ * The current entry guard selection code is designed to try to avoid
+ * _ever_ trying every guard on the network, to try to stick to guards
+ * that we've used before, to handle hostile/broken networks, and
+ * to behave sanely when the network goes up and down.
+ *
+ * Our algorithm works as follows: First, we maintain a SAMPLE of guards
+ * we've seen in the networkstatus consensus.  We maintain this sample
+ * over time, and store it persistently; it is chosen without reference
+ * to our configuration or firewall rules.  Guards remain in the sample
+ * as they enter and leave the consensus.  We expand this sample as
+ * needed, up to a maximum size.
+ *
+ * As a subset of the sample, we maintain a FILTERED SET of the guards
+ * that we would be willing to use if we could connect to them.  The
+ * filter removes all the guards that we're excluding because they're
+ * bridges (or not bridges), because we have restrictive firewall rules,
+ * because of ExcludeNodes, because we of path bias restrictions,
+ * because they're absent from the network at present, and so on.
+ *
+ * As a subset of the filtered set, we keep a REACHABLE FILTERED SET
+ * (also called a "usable filtered set") of those guards that we call
+ * "reachable" or "maybe reachable".  A guard is reachable if we've
+ * connected to it more recently than we've failed.  A guard is "maybe
+ * reachable" if we have never tried to connect to it, or if we
+ * failed to connect to it so long ago that we no longer think our
+ * failure means it's down.
+ *
+ * As a persistent ordered list whose elements are taken from the
+ * sampled set, we track a CONFIRMED GUARDS LIST.  A guard becomes
+ * confirmed when we successfully build a circuit through it, and decide
+ * to use that circuit.  We order the guards on this list by the order
+ * in which they became confirmed.
+ *
+ * And as a final group, we have an ordered list of PRIMARY GUARDS,
+ * whose elements are taken from the filtered set. We prefer
+ * confirmed guards to non-confirmed guards for this list, and place
+ * other restrictions on it.  The primary guards are the ones that we
+ * connect to "when nothing is wrong" -- circuits through them can be used
+ * immediately.
+ *
+ * To build circuits, we take a primary guard if possible -- or a
+ * reachable filtered confirmed guard if no primary guard is possible --
+ * or a random reachable filtered guard otherwise.  If the guard is
+ * primary, we can use the circuit immediately on success.  Otherwise,
+ * the guard is now "pending" -- we won't use its circuit unless all
+ * of the circuits we're trying to build through better guards have
+ * definitely failed.
+ *
+ * While we're building circuits, we track a little "guard state" for
+ * each circuit. We use this to keep track of whether the circuit is
+ * one that we can use as soon as it's done, or whether it's one that
+ * we should keep around to see if we can do better.  In the latter case,
+ * a periodic call to entry_guards_upgrade_waiting_circuits() will
+ * eventually upgrade it.
+ **/
+/* DOCDOC -- expand this.
+ *
+ * Information invariants:
+ *
+ * [x] whenever a guard becomes unreachable, clear its usable_filtered flag.
+ *
+ * [x] Whenever a guard becomes reachable or maybe-reachable, if its filtered
+ * flag is set, set its usable_filtered flag.
+ *
+ * [x] Whenever we get a new consensus, call update_from_consensus(). (LATER.)
+ *
+ * [x] Whenever the configuration changes in a relevant way, update the
+ * filtered/usable flags. (LATER.)
+ *
+ * [x] Whenever we add a guard to the sample, make sure its filtered/usable
+ * flags are set as possible.
+ *
+ * [x] Whenever we remove a guard from the sample, remove it from the primary
+ * and confirmed lists.
+ *
+ * [x] When we make a guard confirmed, update the primary list.
+ *
+ * [x] When we make a guard filtered or unfiltered, update the primary list.
+ *
+ * [x] When we are about to pick a guard, make sure that the primary list is
+ * full.
+ *
+ * [x] Before calling sample_reachable_filtered_entry_guards(), make sure
+ * that the filtered, primary, and confirmed flags are up-to-date.
+ *
+ * [x] Call entry_guard_consider_retry every time we are about to check
+ * is_usable_filtered or is_reachable, and every time we set
+ * is_filtered to 1.
+ *
+ * [x] Call entry_guards_changed_for_guard_selection() whenever we update
+ * a persistent field.
+ */
+
+#define ENTRYNODES_PRIVATE
+
+#include "or/or.h"
+#include "or/channel.h"
+#include "or/bridges.h"
+#include "or/circpathbias.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/circuitstats.h"
+#include "or/config.h"
+#include "or/confparse.h"
+#include "or/connection.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "or/directory.h"
+#include "or/entrynodes.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+#include "or/transports.h"
+#include "or/statefile.h"
+#include "lib/math/fp.h"
+#include "lib/encoding/confline.h"
+
+#include "or/node_st.h"
+#include "or/origin_circuit_st.h"
+#include "or/or_state_st.h"
+
+#include "lib/crypt_ops/digestset.h"
+
+/** A list of existing guard selection contexts. */
+static smartlist_t *guard_contexts = NULL;
+/** The currently enabled guard selection context. */
+static guard_selection_t *curr_guard_context = NULL;
+
+/** A value of 1 means that at least one context has changed,
+ * and those changes need to be flushed to disk. */
+static int entry_guards_dirty = 0;
+
+static void entry_guard_set_filtered_flags(const or_options_t *options,
+                                           guard_selection_t *gs,
+                                           entry_guard_t *guard);
+static void pathbias_check_use_success_count(entry_guard_t *guard);
+static void pathbias_check_close_success_count(entry_guard_t *guard);
+static int node_is_possible_guard(const node_t *node);
+static int node_passes_guard_filter(const or_options_t *options,
+                                    const node_t *node);
+static entry_guard_t *entry_guard_add_to_sample_impl(guard_selection_t *gs,
+                               const uint8_t *rsa_id_digest,
+                               const char *nickname,
+                               const tor_addr_port_t *bridge_addrport);
+static entry_guard_t *get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
+                                              const tor_addr_port_t *addrport);
+static int entry_guard_obeys_restriction(const entry_guard_t *guard,
+                                         const entry_guard_restriction_t *rst);
+
+/** Return 0 if we should apply guardfraction information found in the
+ *  consensus. A specific consensus can be specified with the
+ *  <b>ns</b> argument, if NULL the most recent one will be picked.*/
+int
+should_apply_guardfraction(const networkstatus_t *ns)
+{
+  /* We need to check the corresponding torrc option and the consensus
+   * parameter if we need to. */
+  const or_options_t *options = get_options();
+
+  /* If UseGuardFraction is 'auto' then check the same-named consensus
+   * parameter. If the consensus parameter is not present, default to
+   * "off". */
+  if (options->UseGuardFraction == -1) {
+    return networkstatus_get_param(ns, "UseGuardFraction",
+                                   0, /* default to "off" */
+                                   0, 1);
+  }
+
+  return options->UseGuardFraction;
+}
+
+/** Return true iff we know a preferred descriptor for <b>guard</b> */
+static int
+guard_has_descriptor(const entry_guard_t *guard)
+{
+  const node_t *node = node_get_by_id(guard->identity);
+  if (!node)
+    return 0;
+  return node_has_preferred_descriptor(node, 1);
+}
+
+/**
+ * Try to determine the correct type for a selection named "name",
+ * if <b>type</b> is GS_TYPE_INFER.
+ */
+STATIC guard_selection_type_t
+guard_selection_infer_type(guard_selection_type_t type,
+                           const char *name)
+{
+  if (type == GS_TYPE_INFER) {
+    if (!strcmp(name, "bridges"))
+      type = GS_TYPE_BRIDGE;
+    else if (!strcmp(name, "restricted"))
+      type = GS_TYPE_RESTRICTED;
+    else
+      type = GS_TYPE_NORMAL;
+  }
+  return type;
+}
+
+/**
+ * Allocate and return a new guard_selection_t, with the name <b>name</b>.
+ */
+STATIC guard_selection_t *
+guard_selection_new(const char *name,
+                    guard_selection_type_t type)
+{
+  guard_selection_t *gs;
+
+  type = guard_selection_infer_type(type, name);
+
+  gs = tor_malloc_zero(sizeof(*gs));
+  gs->name = tor_strdup(name);
+  gs->type = type;
+  gs->sampled_entry_guards = smartlist_new();
+  gs->confirmed_entry_guards = smartlist_new();
+  gs->primary_entry_guards = smartlist_new();
+
+  return gs;
+}
+
+/**
+ * Return the guard selection called <b>name</b>. If there is none, and
+ * <b>create_if_absent</b> is true, then create and return it.  If there
+ * is none, and <b>create_if_absent</b> is false, then return NULL.
+ */
+STATIC guard_selection_t *
+get_guard_selection_by_name(const char *name,
+                            guard_selection_type_t type,
+                            int create_if_absent)
+{
+  if (!guard_contexts) {
+    guard_contexts = smartlist_new();
+  }
+  SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    if (!strcmp(gs->name, name))
+      return gs;
+  } SMARTLIST_FOREACH_END(gs);
+
+  if (! create_if_absent)
+    return NULL;
+
+  log_debug(LD_GUARD, "Creating a guard selection called %s", name);
+  guard_selection_t *new_selection = guard_selection_new(name, type);
+  smartlist_add(guard_contexts, new_selection);
+
+  return new_selection;
+}
+
+/**
+ * Allocate the first guard context that we're planning to use,
+ * and make it the current context.
+ */
+static void
+create_initial_guard_context(void)
+{
+  tor_assert(! curr_guard_context);
+  if (!guard_contexts) {
+    guard_contexts = smartlist_new();
+  }
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *name = choose_guard_selection(
+                             get_options(),
+                             networkstatus_get_live_consensus(approx_time()),
+                             NULL,
+                             &type);
+  tor_assert(name); // "name" can only be NULL if we had an old name.
+  tor_assert(type != GS_TYPE_INFER);
+  log_notice(LD_GUARD, "Starting with guard context \"%s\"", name);
+  curr_guard_context = get_guard_selection_by_name(name, type, 1);
+}
+
+/** Get current default guard_selection_t, creating it if necessary */
+guard_selection_t *
+get_guard_selection_info(void)
+{
+  if (!curr_guard_context) {
+    create_initial_guard_context();
+  }
+
+  return curr_guard_context;
+}
+
+/** Return a statically allocated human-readable description of <b>guard</b>
+ */
+const char *
+entry_guard_describe(const entry_guard_t *guard)
+{
+  static char buf[256];
+  tor_snprintf(buf, sizeof(buf),
+               "%s ($%s)",
+               strlen(guard->nickname) ? guard->nickname : "[bridge]",
+               hex_str(guard->identity, DIGEST_LEN));
+  return buf;
+}
+
+/** Return <b>guard</b>'s 20-byte RSA identity digest */
+const char *
+entry_guard_get_rsa_id_digest(const entry_guard_t *guard)
+{
+  return guard->identity;
+}
+
+/** Return the pathbias state associated with <b>guard</b>. */
+guard_pathbias_t *
+entry_guard_get_pathbias_state(entry_guard_t *guard)
+{
+  return &guard->pb;
+}
+
+HANDLE_IMPL(entry_guard, entry_guard_t, ATTR_UNUSED STATIC)
+
+/** Return an interval betweeen 'now' and 'max_backdate' seconds in the past,
+ * chosen uniformly at random.  We use this before recording persistent
+ * dates, so that we aren't leaking exactly when we recorded it.
+ */
+MOCK_IMPL(STATIC time_t,
+randomize_time,(time_t now, time_t max_backdate))
+{
+  tor_assert(max_backdate > 0);
+
+  time_t earliest = now - max_backdate;
+  time_t latest = now;
+  if (earliest <= 0)
+    earliest = 1;
+  if (latest <= earliest)
+    latest = earliest + 1;
+
+  return crypto_rand_time_range(earliest, latest);
+}
+
+/**
+ * @name parameters for networkstatus algorithm
+ *
+ * These parameters are taken from the consensus; some are overrideable in
+ * the torrc.
+ */
+/**@{*/
+/**
+ * We never let our sampled guard set grow larger than this fraction
+ * of the guards on the network.
+ */
+STATIC double
+get_max_sample_threshold(void)
+{
+  int32_t pct =
+    networkstatus_get_param(NULL, "guard-max-sample-threshold-percent",
+                            DFLT_MAX_SAMPLE_THRESHOLD_PERCENT,
+                            1, 100);
+  return pct / 100.0;
+}
+/**
+ * We never let our sampled guard set grow larger than this number.
+ */
+STATIC int
+get_max_sample_size_absolute(void)
+{
+  return (int) networkstatus_get_param(NULL, "guard-max-sample-size",
+                                       DFLT_MAX_SAMPLE_SIZE,
+                                       1, INT32_MAX);
+}
+/**
+ * We always try to make our sample contain at least this many guards.
+ */
+STATIC int
+get_min_filtered_sample_size(void)
+{
+  return networkstatus_get_param(NULL, "guard-min-filtered-sample-size",
+                                 DFLT_MIN_FILTERED_SAMPLE_SIZE,
+                                 1, INT32_MAX);
+}
+/**
+ * If a guard is unlisted for this many days in a row, we remove it.
+ */
+STATIC int
+get_remove_unlisted_guards_after_days(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-remove-unlisted-guards-after-days",
+                                 DFLT_REMOVE_UNLISTED_GUARDS_AFTER_DAYS,
+                                 1, 365*10);
+}
+/**
+ * We remove unconfirmed guards from the sample after this many days,
+ * regardless of whether they are listed or unlisted.
+ */
+STATIC int
+get_guard_lifetime(void)
+{
+  if (get_options()->GuardLifetime >= 86400)
+    return get_options()->GuardLifetime;
+  int32_t days;
+  days = networkstatus_get_param(NULL,
+                                 "guard-lifetime-days",
+                                 DFLT_GUARD_LIFETIME_DAYS, 1, 365*10);
+  return days * 86400;
+}
+/**
+ * We remove confirmed guards from the sample if they were sampled
+ * GUARD_LIFETIME_DAYS ago and confirmed this many days ago.
+ */
+STATIC int
+get_guard_confirmed_min_lifetime(void)
+{
+  if (get_options()->GuardLifetime >= 86400)
+    return get_options()->GuardLifetime;
+  int32_t days;
+  days = networkstatus_get_param(NULL, "guard-confirmed-min-lifetime-days",
+                                 DFLT_GUARD_CONFIRMED_MIN_LIFETIME_DAYS,
+                                 1, 365*10);
+  return days * 86400;
+}
+/**
+ * How many guards do we try to keep on our primary guard list?
+ */
+STATIC int
+get_n_primary_guards(void)
+{
+  /* If the user has explicitly configured the number of primary guards, do
+   * what the user wishes to do */
+  const int configured_primaries = get_options()->NumPrimaryGuards;
+  if (configured_primaries) {
+    return configured_primaries;
+  }
+
+  /* otherwise check for consensus parameter and if that's not set either, just
+   * use the default value. */
+  return networkstatus_get_param(NULL,
+                                 "guard-n-primary-guards",
+                                 DFLT_N_PRIMARY_GUARDS, 1, INT32_MAX);
+}
+/**
+ * Return the number of the live primary guards we should look at when
+ * making a circuit.
+ */
+STATIC int
+get_n_primary_guards_to_use(guard_usage_t usage)
+{
+  int configured;
+  const char *param_name;
+  int param_default;
+
+  /* If the user has explicitly configured the amount of guards, use
+     that. Otherwise, fall back to the default value. */
+  if (usage == GUARD_USAGE_DIRGUARD) {
+    configured = get_options()->NumDirectoryGuards;
+    param_name = "guard-n-primary-dir-guards-to-use";
+    param_default = DFLT_N_PRIMARY_DIR_GUARDS_TO_USE;
+  } else {
+    configured = get_options()->NumEntryGuards;
+    param_name = "guard-n-primary-guards-to-use";
+    param_default = DFLT_N_PRIMARY_GUARDS_TO_USE;
+  }
+  if (configured >= 1) {
+    return configured;
+  }
+  return networkstatus_get_param(NULL,
+                                 param_name, param_default, 1, INT32_MAX);
+}
+/**
+ * If we haven't successfully built or used a circuit in this long, then
+ * consider that the internet is probably down.
+ */
+STATIC int
+get_internet_likely_down_interval(void)
+{
+  return networkstatus_get_param(NULL, "guard-internet-likely-down-interval",
+                                 DFLT_INTERNET_LIKELY_DOWN_INTERVAL,
+                                 1, INT32_MAX);
+}
+/**
+ * If we're trying to connect to a nonprimary guard for at least this
+ * many seconds, and we haven't gotten the connection to work, we will treat
+ * lower-priority guards as usable.
+ */
+STATIC int
+get_nonprimary_guard_connect_timeout(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-nonprimary-guard-connect-timeout",
+                                 DFLT_NONPRIMARY_GUARD_CONNECT_TIMEOUT,
+                                 1, INT32_MAX);
+}
+/**
+ * If a circuit has been sitting around in 'waiting for better guard' state
+ * for at least this long, we'll expire it.
+ */
+STATIC int
+get_nonprimary_guard_idle_timeout(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-nonprimary-guard-idle-timeout",
+                                 DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT,
+                                 1, INT32_MAX);
+}
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in a restricted setting.
+ */
+STATIC double
+get_meaningful_restriction_threshold(void)
+{
+  int32_t pct = networkstatus_get_param(NULL,
+                                        "guard-meaningful-restriction-percent",
+                                        DFLT_MEANINGFUL_RESTRICTION_PERCENT,
+                                        1, INT32_MAX);
+  return pct / 100.0;
+}
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in an extremely restricted setting, and should warn.
+ */
+STATIC double
+get_extreme_restriction_threshold(void)
+{
+  int32_t pct = networkstatus_get_param(NULL,
+                                        "guard-extreme-restriction-percent",
+                                        DFLT_EXTREME_RESTRICTION_PERCENT,
+                                        1, INT32_MAX);
+  return pct / 100.0;
+}
+
+/* Mark <b>guard</b> as maybe reachable again. */
+static void
+mark_guard_maybe_reachable(entry_guard_t *guard)
+{
+  if (guard->is_reachable != GUARD_REACHABLE_NO) {
+    return;
+  }
+
+  /* Note that we do not clear failing_since: this guard is now only
+   * _maybe-reachable_. */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+  if (guard->is_filtered_guard)
+    guard->is_usable_filtered_guard = 1;
+}
+
+/**
+ * Called when the network comes up after having seemed to be down for
+ * a while: Mark the primary guards as maybe-reachable so that we'll
+ * try them again.
+ */
+STATIC void
+mark_primary_guards_maybe_reachable(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    mark_guard_maybe_reachable(guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/* Called when we exhaust all guards in our sampled set: Marks all guards as
+   maybe-reachable so that we 'll try them again. */
+static void
+mark_all_guards_maybe_reachable(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    mark_guard_maybe_reachable(guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/**@}*/
+
+/**
+ * Given our options and our list of nodes, return the name of the
+ * guard selection that we should use.  Return NULL for "use the
+ * same selection you were using before.
+ */
+STATIC const char *
+choose_guard_selection(const or_options_t *options,
+                       const networkstatus_t *live_ns,
+                       const guard_selection_t *old_selection,
+                       guard_selection_type_t *type_out)
+{
+  tor_assert(options);
+  tor_assert(type_out);
+
+  if (options->UseBridges) {
+    *type_out = GS_TYPE_BRIDGE;
+    return "bridges";
+  }
+
+  if (! live_ns) {
+    /* without a networkstatus, we can't tell any more than that. */
+    *type_out = GS_TYPE_NORMAL;
+    return "default";
+  }
+
+  const smartlist_t *nodes = nodelist_get_list();
+  int n_guards = 0, n_passing_filter = 0;
+  SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+    if (node_is_possible_guard(node)) {
+      ++n_guards;
+      if (node_passes_guard_filter(options, node)) {
+        ++n_passing_filter;
+      }
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* We use separate 'high' and 'low' thresholds here to prevent flapping
+   * back and forth */
+  const int meaningful_threshold_high =
+    (int)(n_guards * get_meaningful_restriction_threshold() * 1.05);
+  const int meaningful_threshold_mid =
+    (int)(n_guards * get_meaningful_restriction_threshold());
+  const int meaningful_threshold_low =
+    (int)(n_guards * get_meaningful_restriction_threshold() * .95);
+  const int extreme_threshold =
+    (int)(n_guards * get_extreme_restriction_threshold());
+
+  /*
+    If we have no previous selection, then we're "restricted" iff we are
+    below the meaningful restriction threshold.  That's easy enough.
+
+    But if we _do_ have a previous selection, we make it a little
+    "sticky": we only move from "restricted" to "default" when we find
+    that we're above the threshold plus 5%, and we only move from
+    "default" to "restricted" when we're below the threshold minus 5%.
+    That should prevent us from flapping back and forth if we happen to
+    be hovering very close to the default.
+
+    The extreme threshold is for warning only.
+  */
+
+  static int have_warned_extreme_threshold = 0;
+  if (n_guards &&
+      n_passing_filter < extreme_threshold &&
+      ! have_warned_extreme_threshold) {
+    have_warned_extreme_threshold = 1;
+    const double exclude_frac =
+      (n_guards - n_passing_filter) / (double)n_guards;
+    log_warn(LD_GUARD, "Your configuration excludes %d%% of all possible "
+             "guards. That's likely to make you stand out from the "
+             "rest of the world.", (int)(exclude_frac * 100));
+  }
+
+  /* Easy case: no previous selection. Just check if we are in restricted or
+     normal guard selection. */
+  if (old_selection == NULL) {
+    if (n_passing_filter >= meaningful_threshold_mid) {
+      *type_out = GS_TYPE_NORMAL;
+      return "default";
+    } else {
+      *type_out = GS_TYPE_RESTRICTED;
+      return "restricted";
+    }
+  }
+
+  /* Trickier case: we do have a previous guard selection context. */
+  tor_assert(old_selection);
+
+  /* Use high and low thresholds to decide guard selection, and if we fall in
+     the middle then keep the current guard selection context. */
+  if (n_passing_filter >= meaningful_threshold_high) {
+    *type_out = GS_TYPE_NORMAL;
+    return "default";
+  } else if (n_passing_filter < meaningful_threshold_low) {
+    *type_out = GS_TYPE_RESTRICTED;
+    return "restricted";
+  } else {
+    /* we are in the middle: maintain previous guard selection */
+    *type_out = old_selection->type;
+    return old_selection->name;
+  }
+}
+
+/**
+ * Check whether we should switch from our current guard selection to a
+ * different one.  If so, switch and return 1.  Return 0 otherwise.
+ *
+ * On a 1 return, the caller should mark all currently live circuits unusable
+ * for new streams, by calling circuit_mark_all_unused_circs() and
+ * circuit_mark_all_dirty_circs_as_unusable().
+ */
+int
+update_guard_selection_choice(const or_options_t *options)
+{
+  if (!curr_guard_context) {
+    create_initial_guard_context();
+    return 1;
+  }
+
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *new_name = choose_guard_selection(
+                             options,
+                             networkstatus_get_live_consensus(approx_time()),
+                             curr_guard_context,
+                             &type);
+  tor_assert(new_name);
+  tor_assert(type != GS_TYPE_INFER);
+
+  const char *cur_name = curr_guard_context->name;
+  if (! strcmp(cur_name, new_name)) {
+    log_debug(LD_GUARD,
+              "Staying with guard context \"%s\" (no change)", new_name);
+    return 0; // No change
+  }
+
+  log_notice(LD_GUARD, "Switching to guard context \"%s\" (was using \"%s\")",
+             new_name, cur_name);
+  guard_selection_t *new_guard_context;
+  new_guard_context = get_guard_selection_by_name(new_name, type, 1);
+  tor_assert(new_guard_context);
+  tor_assert(new_guard_context != curr_guard_context);
+  curr_guard_context = new_guard_context;
+
+  return 1;
+}
+
+/**
+ * Return true iff <b>node</b> has all the flags needed for us to consider it
+ * a possible guard when sampling guards.
+ */
+static int
+node_is_possible_guard(const node_t *node)
+{
+  /* The "GUARDS" set is all nodes in the nodelist for which this predicate
+   * holds. */
+
+  tor_assert(node);
+  return (node->is_possible_guard &&
+          node->is_stable &&
+          node->is_fast &&
+          node->is_valid &&
+          node_is_dir(node) &&
+          !router_digest_is_me(node->identity));
+}
+
+/**
+ * Return the sampled guard with the RSA identity digest <b>rsa_id</b>, or
+ * NULL if we don't have one. */
+STATIC entry_guard_t *
+get_sampled_guard_with_id(guard_selection_t *gs,
+                          const uint8_t *rsa_id)
+{
+  tor_assert(gs);
+  tor_assert(rsa_id);
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    if (tor_memeq(guard->identity, rsa_id, DIGEST_LEN))
+      return guard;
+  } SMARTLIST_FOREACH_END(guard);
+  return NULL;
+}
+
+/** If <b>gs</b> contains a sampled entry guard matching <b>bridge</b>,
+ * return that guard. Otherwise return NULL. */
+static entry_guard_t *
+get_sampled_guard_for_bridge(guard_selection_t *gs,
+                             const bridge_info_t *bridge)
+{
+  const uint8_t *id = bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+  entry_guard_t *guard;
+  if (BUG(!addrport))
+    return NULL; // LCOV_EXCL_LINE
+  guard = get_sampled_guard_by_bridge_addr(gs, addrport);
+  if (! guard || (id && tor_memneq(id, guard->identity, DIGEST_LEN)))
+    return NULL;
+  else
+    return guard;
+}
+
+/** If we know a bridge_info_t matching <b>guard</b>, return that
+ * bridge.  Otherwise return NULL. */
+static bridge_info_t *
+get_bridge_info_for_guard(const entry_guard_t *guard)
+{
+  const uint8_t *identity = NULL;
+  if (! tor_digest_is_zero(guard->identity)) {
+    identity = (const uint8_t *)guard->identity;
+  }
+  if (BUG(guard->bridge_addr == NULL))
+    return NULL;
+
+  return get_configured_bridge_by_exact_addr_port_digest(
+                                                 &guard->bridge_addr->addr,
+                                                 guard->bridge_addr->port,
+                                                 (const char*)identity);
+}
+
+/**
+ * Return true iff we have a sampled guard with the RSA identity digest
+ * <b>rsa_id</b>. */
+static inline int
+have_sampled_guard_with_id(guard_selection_t *gs, const uint8_t *rsa_id)
+{
+  return get_sampled_guard_with_id(gs, rsa_id) != NULL;
+}
+
+/**
+ * Allocate a new entry_guard_t object for <b>node</b>, add it to the
+ * sampled entry guards in <b>gs</b>, and return it. <b>node</b> must
+ * not currently be a sampled guard in <b>gs</b>.
+ */
+STATIC entry_guard_t *
+entry_guard_add_to_sample(guard_selection_t *gs,
+                          const node_t *node)
+{
+  log_info(LD_GUARD, "Adding %s to the entry guard sample set.",
+           node_describe(node));
+
+  /* make sure that the guard is not already sampled. */
+  if (BUG(have_sampled_guard_with_id(gs, (const uint8_t*)node->identity)))
+    return NULL; // LCOV_EXCL_LINE
+
+  return entry_guard_add_to_sample_impl(gs,
+                                        (const uint8_t*)node->identity,
+                                        node_get_nickname(node),
+                                        NULL);
+}
+
+/**
+ * Backend: adds a new sampled guard to <b>gs</b>, with given identity,
+ * nickname, and ORPort.  rsa_id_digest and bridge_addrport are optional, but
+ * we need one of them. nickname is optional. The caller is responsible for
+ * maintaining the size limit of the SAMPLED_GUARDS set.
+ */
+static entry_guard_t *
+entry_guard_add_to_sample_impl(guard_selection_t *gs,
+                               const uint8_t *rsa_id_digest,
+                               const char *nickname,
+                               const tor_addr_port_t *bridge_addrport)
+{
+  const int GUARD_LIFETIME = get_guard_lifetime();
+  tor_assert(gs);
+
+  // XXXX #20827 take ed25519 identity here too.
+
+  /* Make sure we can actually identify the guard. */
+  if (BUG(!rsa_id_digest && !bridge_addrport))
+    return NULL; // LCOV_EXCL_LINE
+
+  entry_guard_t *guard = tor_malloc_zero(sizeof(entry_guard_t));
+
+  /* persistent fields */
+  guard->is_persistent = (rsa_id_digest != NULL);
+  guard->selection_name = tor_strdup(gs->name);
+  if (rsa_id_digest)
+    memcpy(guard->identity, rsa_id_digest, DIGEST_LEN);
+  if (nickname)
+    strlcpy(guard->nickname, nickname, sizeof(guard->nickname));
+  guard->sampled_on_date = randomize_time(approx_time(), GUARD_LIFETIME/10);
+  tor_free(guard->sampled_by_version);
+  guard->sampled_by_version = tor_strdup(VERSION);
+  guard->currently_listed = 1;
+  guard->confirmed_idx = -1;
+
+  /* non-persistent fields */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+  if (bridge_addrport)
+    guard->bridge_addr = tor_memdup(bridge_addrport, sizeof(*bridge_addrport));
+
+  smartlist_add(gs->sampled_entry_guards, guard);
+  guard->in_selection = gs;
+  entry_guard_set_filtered_flags(get_options(), gs, guard);
+  entry_guards_changed_for_guard_selection(gs);
+  return guard;
+}
+
+/**
+ * Add an entry guard to the "bridges" guard selection sample, with
+ * information taken from <b>bridge</b>. Return that entry guard.
+ */
+static entry_guard_t *
+entry_guard_add_bridge_to_sample(guard_selection_t *gs,
+                                 const bridge_info_t *bridge)
+{
+  const uint8_t *id_digest = bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+
+  tor_assert(addrport);
+
+  /* make sure that the guard is not already sampled. */
+  if (BUG(get_sampled_guard_for_bridge(gs, bridge)))
+    return NULL; // LCOV_EXCL_LINE
+
+  return entry_guard_add_to_sample_impl(gs, id_digest, NULL, addrport);
+}
+
+/**
+ * Return the entry_guard_t in <b>gs</b> whose address is <b>addrport</b>,
+ * or NULL if none exists.
+*/
+static entry_guard_t *
+get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
+                                 const tor_addr_port_t *addrport)
+{
+  if (! gs)
+    return NULL;
+  if (BUG(!addrport))
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, g) {
+    if (g->bridge_addr && tor_addr_port_eq(addrport, g->bridge_addr))
+      return g;
+  } SMARTLIST_FOREACH_END(g);
+  return NULL;
+}
+
+/** Update the guard subsystem's knowledge of the identity of the bridge
+ * at <b>addrport</b>.  Idempotent.
+ */
+void
+entry_guard_learned_bridge_identity(const tor_addr_port_t *addrport,
+                                    const uint8_t *rsa_id_digest)
+{
+  guard_selection_t *gs = get_guard_selection_by_name("bridges",
+                                                      GS_TYPE_BRIDGE,
+                                                      0);
+  if (!gs)
+    return;
+
+  entry_guard_t *g = get_sampled_guard_by_bridge_addr(gs, addrport);
+  if (!g)
+    return;
+
+  int make_persistent = 0;
+
+  if (tor_digest_is_zero(g->identity)) {
+    memcpy(g->identity, rsa_id_digest, DIGEST_LEN);
+    make_persistent = 1;
+  } else if (tor_memeq(g->identity, rsa_id_digest, DIGEST_LEN)) {
+    /* Nothing to see here; we learned something we already knew. */
+    if (BUG(! g->is_persistent))
+      make_persistent = 1;
+  } else {
+    char old_id[HEX_DIGEST_LEN+1];
+    base16_encode(old_id, sizeof(old_id), g->identity, sizeof(g->identity));
+    log_warn(LD_BUG, "We 'learned' an identity %s for a bridge at %s:%d, but "
+             "we already knew a different one (%s). Ignoring the new info as "
+             "possibly bogus.",
+             hex_str((const char *)rsa_id_digest, DIGEST_LEN),
+             fmt_and_decorate_addr(&addrport->addr), addrport->port,
+             old_id);
+    return; // redundant, but let's be clear: we're not making this persistent.
+  }
+
+  if (make_persistent) {
+    g->is_persistent = 1;
+    entry_guards_changed_for_guard_selection(gs);
+  }
+}
+
+/**
+ * Return the number of sampled guards in <b>gs</b> that are "filtered"
+ * (that is, we're willing to connect to them) and that are "usable"
+ * (that is, either "reachable" or "maybe reachable").
+ *
+ * If a restriction is provided in <b>rst</b>, do not count any guards that
+ * violate it.
+ */
+STATIC int
+num_reachable_filtered_guards(const guard_selection_t *gs,
+                              const entry_guard_restriction_t *rst)
+{
+  int n_reachable_filtered_guards = 0;
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (guard->is_usable_filtered_guard)
+      ++n_reachable_filtered_guards;
+  } SMARTLIST_FOREACH_END(guard);
+  return n_reachable_filtered_guards;
+}
+
+/** Return the actual maximum size for the sample in <b>gs</b>,
+ * given that we know about <b>n_guards</b> total. */
+static int
+get_max_sample_size(guard_selection_t *gs,
+                    int n_guards)
+{
+  const int using_bridges = (gs->type == GS_TYPE_BRIDGE);
+  const int min_sample = get_min_filtered_sample_size();
+
+  /* If we are in bridge mode, expand our sample set as needed without worrying
+   * about max size. We should respect the user's wishes to use many bridges if
+   * that's what they have specified in their configuration file. */
+  if (using_bridges)
+    return INT_MAX;
+
+  const int max_sample_by_pct = (int)(n_guards * get_max_sample_threshold());
+  const int max_sample_absolute = get_max_sample_size_absolute();
+  const int max_sample = MIN(max_sample_by_pct, max_sample_absolute);
+  if (max_sample < min_sample)
+    return min_sample;
+  else
+    return max_sample;
+}
+
+/**
+ * Return a smartlist of the all the guards that are not currently
+ * members of the sample (GUARDS - SAMPLED_GUARDS).  The elements of
+ * this list are node_t pointers in the non-bridge case, and
+ * bridge_info_t pointers in the bridge case.  Set *<b>n_guards_out/b>
+ * to the number of guards that we found in GUARDS, including those
+ * that were already sampled.
+ */
+static smartlist_t *
+get_eligible_guards(const or_options_t *options,
+                    guard_selection_t *gs,
+                    int *n_guards_out)
+{
+  /* Construct eligible_guards as GUARDS - SAMPLED_GUARDS */
+  smartlist_t *eligible_guards = smartlist_new();
+  int n_guards = 0; // total size of "GUARDS"
+
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const smartlist_t *bridges = bridge_list_get();
+    SMARTLIST_FOREACH_BEGIN(bridges, bridge_info_t *, bridge) {
+      ++n_guards;
+      if (NULL != get_sampled_guard_for_bridge(gs, bridge)) {
+        continue;
+      }
+      smartlist_add(eligible_guards, bridge);
+    } SMARTLIST_FOREACH_END(bridge);
+  } else {
+    const smartlist_t *nodes = nodelist_get_list();
+    const int n_sampled = smartlist_len(gs->sampled_entry_guards);
+
+    /* Build a bloom filter of our current guards: let's keep this O(N). */
+    digestset_t *sampled_guard_ids = digestset_new(n_sampled);
+    SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, const entry_guard_t *,
+                            guard) {
+      digestset_add(sampled_guard_ids, guard->identity);
+    } SMARTLIST_FOREACH_END(guard);
+
+    SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+      if (! node_is_possible_guard(node))
+        continue;
+      if (gs->type == GS_TYPE_RESTRICTED) {
+        /* In restricted mode, we apply the filter BEFORE sampling, so
+         * that we are sampling from the nodes that we might actually
+         * select. If we sampled first, we might wind up with a sample
+         * that didn't include any EntryNodes at all. */
+        if (! node_passes_guard_filter(options, node))
+          continue;
+      }
+      ++n_guards;
+      if (digestset_probably_contains(sampled_guard_ids, node->identity))
+        continue;
+      smartlist_add(eligible_guards, (node_t*)node);
+    } SMARTLIST_FOREACH_END(node);
+
+    /* Now we can free that bloom filter. */
+    digestset_free(sampled_guard_ids);
+  }
+
+  *n_guards_out = n_guards;
+  return eligible_guards;
+}
+
+/** Helper: given a smartlist of either bridge_info_t (if gs->type is
+ * GS_TYPE_BRIDGE) or node_t (otherwise), pick one that can be a guard,
+ * add it as a guard, remove it from the list, and return a new
+ * entry_guard_t.  Return NULL on failure. */
+static entry_guard_t *
+select_and_add_guard_item_for_sample(guard_selection_t *gs,
+                                     smartlist_t *eligible_guards)
+{
+  entry_guard_t *added_guard;
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const bridge_info_t *bridge = smartlist_choose(eligible_guards);
+    if (BUG(!bridge))
+      return NULL; // LCOV_EXCL_LINE
+    smartlist_remove(eligible_guards, bridge);
+    added_guard = entry_guard_add_bridge_to_sample(gs, bridge);
+  } else {
+    const node_t *node =
+      node_sl_choose_by_bandwidth(eligible_guards, WEIGHT_FOR_GUARD);
+    if (BUG(!node))
+      return NULL; // LCOV_EXCL_LINE
+    smartlist_remove(eligible_guards, node);
+    added_guard = entry_guard_add_to_sample(gs, node);
+  }
+
+  return added_guard;
+}
+
+/**
+ * Return true iff we need a consensus to update our guards, but we don't
+ * have one. (We can return 0 here either if the consensus is _not_ missing,
+ * or if we don't need a consensus because we're using bridges.)
+ */
+static int
+live_consensus_is_missing(const guard_selection_t *gs)
+{
+  tor_assert(gs);
+  if (gs->type == GS_TYPE_BRIDGE) {
+    /* We don't update bridges from the consensus; they aren't there. */
+    return 0;
+  }
+  return networkstatus_get_live_consensus(approx_time()) == NULL;
+}
+
+/**
+ * Add new guards to the sampled guards in <b>gs</b> until there are
+ * enough usable filtered guards, but never grow the sample beyond its
+ * maximum size.  Return the last guard added, or NULL if none were
+ * added.
+ */
+STATIC entry_guard_t *
+entry_guards_expand_sample(guard_selection_t *gs)
+{
+  tor_assert(gs);
+  const or_options_t *options = get_options();
+
+  if (live_consensus_is_missing(gs)) {
+    log_info(LD_GUARD, "Not expanding the sample guard set; we have "
+             "no live consensus.");
+    return NULL;
+  }
+
+  int n_sampled = smartlist_len(gs->sampled_entry_guards);
+  entry_guard_t *added_guard = NULL;
+  int n_usable_filtered_guards = num_reachable_filtered_guards(gs, NULL);
+  int n_guards = 0;
+  smartlist_t *eligible_guards = get_eligible_guards(options, gs, &n_guards);
+
+  const int max_sample = get_max_sample_size(gs, n_guards);
+  const int min_filtered_sample = get_min_filtered_sample_size();
+
+  log_info(LD_GUARD, "Expanding the sample guard set. We have %d guards "
+           "in the sample, and %d eligible guards to extend it with.",
+           n_sampled, smartlist_len(eligible_guards));
+
+  while (n_usable_filtered_guards < min_filtered_sample) {
+    /* Has our sample grown too large to expand? */
+    if (n_sampled >= max_sample) {
+      log_info(LD_GUARD, "Not expanding the guard sample any further; "
+               "just hit the maximum sample threshold of %d",
+               max_sample);
+      goto done;
+    }
+
+    /* Did we run out of guards? */
+    if (smartlist_len(eligible_guards) == 0) {
+      /* LCOV_EXCL_START
+         As long as MAX_SAMPLE_THRESHOLD makes can't be adjusted to
+         allow all guards to be sampled, this can't be reached.
+       */
+      log_info(LD_GUARD, "Not expanding the guard sample any further; "
+               "just ran out of eligible guards");
+      goto done;
+      /* LCOV_EXCL_STOP */
+    }
+
+    /* Otherwise we can add at least one new guard. */
+    added_guard = select_and_add_guard_item_for_sample(gs, eligible_guards);
+    if (!added_guard)
+      goto done; // LCOV_EXCL_LINE -- only fails on BUG.
+
+    ++n_sampled;
+
+    if (added_guard->is_usable_filtered_guard)
+      ++n_usable_filtered_guards;
+  }
+
+ done:
+  smartlist_free(eligible_guards);
+  return added_guard;
+}
+
+/**
+ * Helper: <b>guard</b> has just been removed from the sampled guards:
+ * also remove it from primary and confirmed. */
+static void
+remove_guard_from_confirmed_and_primary_lists(guard_selection_t *gs,
+                                              entry_guard_t *guard)
+{
+  if (guard->is_primary) {
+    guard->is_primary = 0;
+    smartlist_remove_keeporder(gs->primary_entry_guards, guard);
+  } else {
+    if (BUG(smartlist_contains(gs->primary_entry_guards, guard))) {
+      smartlist_remove_keeporder(gs->primary_entry_guards, guard);
+    }
+  }
+
+  if (guard->confirmed_idx >= 0) {
+    smartlist_remove_keeporder(gs->confirmed_entry_guards, guard);
+    guard->confirmed_idx = -1;
+    guard->confirmed_on_date = 0;
+  } else {
+    if (BUG(smartlist_contains(gs->confirmed_entry_guards, guard))) {
+      // LCOV_EXCL_START
+      smartlist_remove_keeporder(gs->confirmed_entry_guards, guard);
+      // LCOV_EXCL_STOP
+    }
+  }
+}
+
+/** Return true iff <b>guard</b> is currently "listed" -- that is, it
+ * appears in the consensus, or as a configured bridge (as
+ * appropriate) */
+MOCK_IMPL(STATIC int,
+entry_guard_is_listed,(guard_selection_t *gs, const entry_guard_t *guard))
+{
+  if (gs->type == GS_TYPE_BRIDGE) {
+    return NULL != get_bridge_info_for_guard(guard);
+  } else {
+    const node_t *node = node_get_by_id(guard->identity);
+
+    return node && node_is_possible_guard(node);
+  }
+}
+
+/**
+ * Update the status of all sampled guards based on the arrival of a
+ * new consensus networkstatus document.  This will include marking
+ * some guards as listed or unlisted, and removing expired guards. */
+STATIC void
+sampled_guards_update_from_consensus(guard_selection_t *gs)
+{
+  tor_assert(gs);
+  const int REMOVE_UNLISTED_GUARDS_AFTER =
+    (get_remove_unlisted_guards_after_days() * 86400);
+  const int unlisted_since_slop = REMOVE_UNLISTED_GUARDS_AFTER / 5;
+
+  // It's important to use only a live consensus here; we don't want to
+  // make changes based on anything expired or old.
+  if (live_consensus_is_missing(gs)) {
+    log_info(LD_GUARD, "Not updating the sample guard set; we have "
+             "no live consensus.");
+    return;
+  }
+  log_info(LD_GUARD, "Updating sampled guard status based on received "
+           "consensus.");
+
+  int n_changes = 0;
+
+  /* First: Update listed/unlisted. */
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    /* XXXX #20827 check ed ID too */
+    const int is_listed = entry_guard_is_listed(gs, guard);
+
+    if (is_listed && ! guard->currently_listed) {
+      ++n_changes;
+      guard->currently_listed = 1;
+      guard->unlisted_since_date = 0;
+      log_info(LD_GUARD, "Sampled guard %s is now listed again.",
+               entry_guard_describe(guard));
+    } else if (!is_listed && guard->currently_listed) {
+      ++n_changes;
+      guard->currently_listed = 0;
+      guard->unlisted_since_date = randomize_time(approx_time(),
+                                                  unlisted_since_slop);
+      log_info(LD_GUARD, "Sampled guard %s is now unlisted.",
+               entry_guard_describe(guard));
+    } else if (is_listed && guard->currently_listed) {
+      log_debug(LD_GUARD, "Sampled guard %s is still listed.",
+               entry_guard_describe(guard));
+    } else {
+      tor_assert(! is_listed && ! guard->currently_listed);
+      log_debug(LD_GUARD, "Sampled guard %s is still unlisted.",
+                entry_guard_describe(guard));
+    }
+
+    /* Clean up unlisted_since_date, just in case. */
+    if (guard->currently_listed && guard->unlisted_since_date) {
+      ++n_changes;
+      guard->unlisted_since_date = 0;
+      log_warn(LD_BUG, "Sampled guard %s was listed, but with "
+               "unlisted_since_date set. Fixing.",
+               entry_guard_describe(guard));
+    } else if (!guard->currently_listed && ! guard->unlisted_since_date) {
+      ++n_changes;
+      guard->unlisted_since_date = randomize_time(approx_time(),
+                                                  unlisted_since_slop);
+      log_warn(LD_BUG, "Sampled guard %s was unlisted, but with "
+               "unlisted_since_date unset. Fixing.",
+               entry_guard_describe(guard));
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  const time_t remove_if_unlisted_since =
+    approx_time() - REMOVE_UNLISTED_GUARDS_AFTER;
+  const time_t maybe_remove_if_sampled_before =
+    approx_time() - get_guard_lifetime();
+  const time_t remove_if_confirmed_before =
+    approx_time() - get_guard_confirmed_min_lifetime();
+
+  /* Then: remove the ones that have been junk for too long */
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    int rmv = 0;
+
+    if (guard->currently_listed == 0 &&
+        guard->unlisted_since_date < remove_if_unlisted_since) {
+      /*
+        "We have a live consensus, and {IS_LISTED} is false, and
+         {FIRST_UNLISTED_AT} is over {REMOVE_UNLISTED_GUARDS_AFTER}
+         days in the past."
+      */
+      log_info(LD_GUARD, "Removing sampled guard %s: it has been unlisted "
+               "for over %d days", entry_guard_describe(guard),
+               get_remove_unlisted_guards_after_days());
+      rmv = 1;
+    } else if (guard->sampled_on_date < maybe_remove_if_sampled_before) {
+      /* We have a live consensus, and {ADDED_ON_DATE} is over
+        {GUARD_LIFETIME} ago, *and* {CONFIRMED_ON_DATE} is either
+        "never", or over {GUARD_CONFIRMED_MIN_LIFETIME} ago.
+      */
+      if (guard->confirmed_on_date == 0) {
+        rmv = 1;
+        log_info(LD_GUARD, "Removing sampled guard %s: it was sampled "
+                 "over %d days ago, but never confirmed.",
+                 entry_guard_describe(guard),
+                 get_guard_lifetime() / 86400);
+      } else if (guard->confirmed_on_date < remove_if_confirmed_before) {
+        rmv = 1;
+        log_info(LD_GUARD, "Removing sampled guard %s: it was sampled "
+                 "over %d days ago, and confirmed over %d days ago.",
+                 entry_guard_describe(guard),
+                 get_guard_lifetime() / 86400,
+                 get_guard_confirmed_min_lifetime() / 86400);
+      }
+    }
+
+    if (rmv) {
+      ++n_changes;
+      SMARTLIST_DEL_CURRENT(gs->sampled_entry_guards, guard);
+      remove_guard_from_confirmed_and_primary_lists(gs, guard);
+      entry_guard_free(guard);
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  if (n_changes) {
+    gs->primary_guards_up_to_date = 0;
+    entry_guards_update_filtered_sets(gs);
+    /* We don't need to rebuild the confirmed list right here -- we may have
+     * removed confirmed guards above, but we can't have added any new
+     * confirmed guards.
+     */
+    entry_guards_changed_for_guard_selection(gs);
+  }
+}
+
+/**
+ * Return true iff <b>node</b> is a Tor relay that we are configured to
+ * be able to connect to. */
+static int
+node_passes_guard_filter(const or_options_t *options,
+                         const node_t *node)
+{
+  /* NOTE: Make sure that this function stays in sync with
+   * options_transition_affects_entry_guards */
+  if (routerset_contains_node(options->ExcludeNodes, node))
+    return 0;
+
+  if (options->EntryNodes &&
+      !routerset_contains_node(options->EntryNodes, node))
+    return 0;
+
+  if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0))
+    return 0;
+
+  if (node_is_a_configured_bridge(node))
+    return 0;
+
+  return 1;
+}
+
+/** Helper: Return true iff <b>bridge</b> passes our configuration
+ * filter-- if it is a relay that we are configured to be able to
+ * connect to. */
+static int
+bridge_passes_guard_filter(const or_options_t *options,
+                           const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  if (!bridge)
+    return 0;
+
+  if (routerset_contains_bridge(options->ExcludeNodes, bridge))
+    return 0;
+
+  /* Ignore entrynodes */
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+
+  if (!fascist_firewall_allows_address_addr(&addrport->addr,
+                                            addrport->port,
+                                            FIREWALL_OR_CONNECTION,
+                                            0, 0))
+    return 0;
+
+  return 1;
+}
+
+/**
+ * Return true iff <b>guard</b> is a Tor relay that we are configured to
+ * be able to connect to, and we haven't disabled it for omission from
+ * the consensus or path bias issues. */
+static int
+entry_guard_passes_filter(const or_options_t *options, guard_selection_t *gs,
+                          entry_guard_t *guard)
+{
+  if (guard->currently_listed == 0)
+    return 0;
+  if (guard->pb.path_bias_disabled)
+    return 0;
+
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const bridge_info_t *bridge = get_bridge_info_for_guard(guard);
+    if (bridge == NULL)
+      return 0;
+    return bridge_passes_guard_filter(options, bridge);
+  } else {
+    const node_t *node = node_get_by_id(guard->identity);
+    if (node == NULL) {
+      // This can happen when currently_listed is true, and we're not updating
+      // it because we don't have a live consensus.
+      return 0;
+    }
+
+    return node_passes_guard_filter(options, node);
+  }
+}
+
+/** Return true iff <b>guard</b> is in the same family as <b>node</b>.
+ */
+static int
+guard_in_node_family(const entry_guard_t *guard, const node_t *node)
+{
+  const node_t *guard_node = node_get_by_id(guard->identity);
+  if (guard_node) {
+    return nodes_in_same_family(guard_node, node);
+  } else {
+    /* If we don't have a node_t for the guard node, we might have
+     * a bridge_info_t for it. So let's check to see whether the bridge
+     * address matches has any family issues.
+     *
+     * (Strictly speaking, I believe this check is unnecessary, since we only
+     * use it to avoid the exit's family when building circuits, and we don't
+     * build multihop circuits until we have a routerinfo_t for the
+     * bridge... at which point, we'll also have a node_t for the
+     * bridge. Nonetheless, it seems wise to include it, in case our
+     * assumptions change down the road.  -nickm.)
+     */
+    if (get_options()->EnforceDistinctSubnets && guard->bridge_addr) {
+      tor_addr_t node_addr;
+      node_get_addr(node, &node_addr);
+      if (addrs_in_same_network_family(&node_addr,
+                                       &guard->bridge_addr->addr)) {
+        return 1;
+      }
+    }
+    return 0;
+  }
+}
+
+/* Allocate and return a new exit guard restriction (where <b>exit_id</b> is of
+ * size DIGEST_LEN) */
+STATIC entry_guard_restriction_t *
+guard_create_exit_restriction(const uint8_t *exit_id)
+{
+  entry_guard_restriction_t *rst = NULL;
+  rst = tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  rst->type = RST_EXIT_NODE;
+  memcpy(rst->exclude_id, exit_id, DIGEST_LEN);
+  return rst;
+}
+
+/** If we have fewer than this many possible usable guards, don't set
+ * MD-availability-based restrictions: we might blacklist all of them. */
+#define MIN_GUARDS_FOR_MD_RESTRICTION 10
+
+/** Return true if we should set md dirserver restrictions. We might not want
+ *  to set those if our guard options are too restricted, since we don't want
+ *  to blacklist all of them. */
+static int
+should_set_md_dirserver_restriction(void)
+{
+  const guard_selection_t *gs = get_guard_selection_info();
+  int num_usable_guards = num_reachable_filtered_guards(gs, NULL);
+
+  /* Don't set restriction if too few reachable filtered guards. */
+  if (num_usable_guards < MIN_GUARDS_FOR_MD_RESTRICTION) {
+    log_info(LD_GUARD, "Not setting md restriction: only %d"
+             " usable guards.", num_usable_guards);
+    return 0;
+  }
+
+  /* We have enough usable guards: set MD restriction */
+  return 1;
+}
+
+/** Allocate and return an outdated md guard restriction. Return NULL if no
+ *  such restriction is needed. */
+STATIC entry_guard_restriction_t *
+guard_create_dirserver_md_restriction(void)
+{
+  entry_guard_restriction_t *rst = NULL;
+
+  if (!should_set_md_dirserver_restriction()) {
+    log_debug(LD_GUARD, "Not setting md restriction: too few "
+              "filtered guards.");
+    return NULL;
+  }
+
+  rst = tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  rst->type = RST_OUTDATED_MD_DIRSERVER;
+
+  return rst;
+}
+
+/* Return True if <b>guard</b> obeys the exit restriction <b>rst</b>. */
+static int
+guard_obeys_exit_restriction(const entry_guard_t *guard,
+                             const entry_guard_restriction_t *rst)
+{
+  tor_assert(rst->type == RST_EXIT_NODE);
+
+  // Exclude the exit ID and all of its family.
+  const node_t *node = node_get_by_id((const char*)rst->exclude_id);
+  if (node && guard_in_node_family(guard, node))
+    return 0;
+
+  return tor_memneq(guard->identity, rst->exclude_id, DIGEST_LEN);
+}
+
+/** Return True if <b>guard</b> should be used as a dirserver for fetching
+ *  microdescriptors. */
+static int
+guard_obeys_md_dirserver_restriction(const entry_guard_t *guard)
+{
+  /* If this guard is an outdated dirserver, don't use it. */
+  if (microdesc_relay_is_outdated_dirserver(guard->identity)) {
+    log_info(LD_GENERAL, "Skipping %s dirserver: outdated",
+             hex_str(guard->identity, DIGEST_LEN));
+    return 0;
+  }
+
+  log_debug(LD_GENERAL, "%s dirserver obeys md restrictions",
+            hex_str(guard->identity, DIGEST_LEN));
+
+  return 1;
+}
+
+/**
+ * Return true iff <b>guard</b> obeys the restrictions defined in <b>rst</b>.
+ * (If <b>rst</b> is NULL, there are no restrictions.)
+ */
+static int
+entry_guard_obeys_restriction(const entry_guard_t *guard,
+                              const entry_guard_restriction_t *rst)
+{
+  tor_assert(guard);
+  if (! rst)
+    return 1; // No restriction?  No problem.
+
+  if (rst->type == RST_EXIT_NODE) {
+    return guard_obeys_exit_restriction(guard, rst);
+  } else if (rst->type == RST_OUTDATED_MD_DIRSERVER) {
+    return guard_obeys_md_dirserver_restriction(guard);
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * Update the <b>is_filtered_guard</b> and <b>is_usable_filtered_guard</b>
+ * flags on <b>guard</b>. */
+void
+entry_guard_set_filtered_flags(const or_options_t *options,
+                               guard_selection_t *gs,
+                               entry_guard_t *guard)
+{
+  unsigned was_filtered = guard->is_filtered_guard;
+  guard->is_filtered_guard = 0;
+  guard->is_usable_filtered_guard = 0;
+
+  if (entry_guard_passes_filter(options, gs, guard)) {
+    guard->is_filtered_guard = 1;
+
+    if (guard->is_reachable != GUARD_REACHABLE_NO)
+      guard->is_usable_filtered_guard = 1;
+
+    entry_guard_consider_retry(guard);
+  }
+  log_debug(LD_GUARD, "Updated sampled guard %s: filtered=%d; "
+            "reachable_filtered=%d.", entry_guard_describe(guard),
+            guard->is_filtered_guard, guard->is_usable_filtered_guard);
+
+  if (!bool_eq(was_filtered, guard->is_filtered_guard)) {
+    /* This guard might now be primary or nonprimary. */
+    gs->primary_guards_up_to_date = 0;
+  }
+}
+
+/**
+ * Update the <b>is_filtered_guard</b> and <b>is_usable_filtered_guard</b>
+ * flag on every guard in <b>gs</b>. */
+STATIC void
+entry_guards_update_filtered_sets(guard_selection_t *gs)
+{
+  const or_options_t *options = get_options();
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_set_filtered_flags(options, gs, guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/**
+ * Return a random guard from the reachable filtered sample guards
+ * in <b>gs</b>, subject to the exclusion rules listed in <b>flags</b>.
+ * Return NULL if no such guard can be found.
+ *
+ * Make sure that the sample is big enough, and that all the filter flags
+ * are set correctly, before calling this function.
+ *
+ * If a restriction is provided in <b>rst</b>, do not return any guards that
+ * violate it.
+ **/
+STATIC entry_guard_t *
+sample_reachable_filtered_entry_guards(guard_selection_t *gs,
+                                       const entry_guard_restriction_t *rst,
+                                       unsigned flags)
+{
+  tor_assert(gs);
+  entry_guard_t *result = NULL;
+  const unsigned exclude_confirmed = flags & SAMPLE_EXCLUDE_CONFIRMED;
+  const unsigned exclude_primary = flags & SAMPLE_EXCLUDE_PRIMARY;
+  const unsigned exclude_pending = flags & SAMPLE_EXCLUDE_PENDING;
+  const unsigned no_update_primary = flags & SAMPLE_NO_UPDATE_PRIMARY;
+  const unsigned need_descriptor = flags & SAMPLE_EXCLUDE_NO_DESCRIPTOR;
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  const int n_reachable_filtered = num_reachable_filtered_guards(gs, rst);
+
+  log_info(LD_GUARD, "Trying to sample a reachable guard: We know of %d "
+           "in the USABLE_FILTERED set.", n_reachable_filtered);
+
+  const int min_filtered_sample = get_min_filtered_sample_size();
+  if (n_reachable_filtered < min_filtered_sample) {
+    log_info(LD_GUARD, "  (That isn't enough. Trying to expand the sample.)");
+    entry_guards_expand_sample(gs);
+  }
+
+  if (exclude_primary && !gs->primary_guards_up_to_date && !no_update_primary)
+    entry_guards_update_primary(gs);
+
+  /* Build the set of reachable filtered guards. */
+  smartlist_t *reachable_filtered_sample = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);// redundant, but cheap.
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (! guard->is_usable_filtered_guard)
+      continue;
+    if (exclude_confirmed && guard->confirmed_idx >= 0)
+      continue;
+    if (exclude_primary && guard->is_primary)
+      continue;
+    if (exclude_pending && guard->is_pending)
+      continue;
+    if (need_descriptor && !guard_has_descriptor(guard))
+      continue;
+    smartlist_add(reachable_filtered_sample, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  log_info(LD_GUARD, "  (After filters [%x], we have %d guards to consider.)",
+           flags, smartlist_len(reachable_filtered_sample));
+
+  if (smartlist_len(reachable_filtered_sample)) {
+    result = smartlist_choose(reachable_filtered_sample);
+    log_info(LD_GUARD, "  (Selected %s.)",
+             result ? entry_guard_describe(result) : "<null>");
+  }
+  smartlist_free(reachable_filtered_sample);
+
+  return result;
+}
+
+/**
+ * Helper: compare two entry_guard_t by their confirmed_idx values.
+ * Used to sort the confirmed list.
+ */
+static int
+compare_guards_by_confirmed_idx(const void **a_, const void **b_)
+{
+  const entry_guard_t *a = *a_, *b = *b_;
+  if (a->confirmed_idx < b->confirmed_idx)
+    return -1;
+  else if (a->confirmed_idx > b->confirmed_idx)
+    return 1;
+  else
+    return 0;
+}
+
+/**
+ * Find the confirmed guards from among the sampled guards in <b>gs</b>,
+ * and put them in confirmed_entry_guards in the correct
+ * order. Recalculate their indices.
+ */
+STATIC void
+entry_guards_update_confirmed(guard_selection_t *gs)
+{
+  smartlist_clear(gs->confirmed_entry_guards);
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    if (guard->confirmed_idx >= 0)
+      smartlist_add(gs->confirmed_entry_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_confirmed_idx);
+
+  int any_changed = 0;
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (guard->confirmed_idx != guard_sl_idx) {
+      any_changed = 1;
+      guard->confirmed_idx = guard_sl_idx;
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  gs->next_confirmed_idx = smartlist_len(gs->confirmed_entry_guards);
+
+  if (any_changed) {
+    entry_guards_changed_for_guard_selection(gs);
+  }
+}
+
+/**
+ * Mark <b>guard</b> as a confirmed guard -- that is, one that we have
+ * connected to, and intend to use again.
+ */
+STATIC void
+make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard)
+{
+  if (BUG(guard->confirmed_on_date && guard->confirmed_idx >= 0))
+    return; // LCOV_EXCL_LINE
+
+  if (BUG(smartlist_contains(gs->confirmed_entry_guards, guard)))
+    return; // LCOV_EXCL_LINE
+
+  const int GUARD_LIFETIME = get_guard_lifetime();
+  guard->confirmed_on_date = randomize_time(approx_time(), GUARD_LIFETIME/10);
+
+  log_info(LD_GUARD, "Marking %s as a confirmed guard (index %d)",
+           entry_guard_describe(guard),
+           gs->next_confirmed_idx);
+
+  guard->confirmed_idx = gs->next_confirmed_idx++;
+  smartlist_add(gs->confirmed_entry_guards, guard);
+
+  // This confirmed guard might kick something else out of the primary
+  // guards.
+  gs->primary_guards_up_to_date = 0;
+
+  entry_guards_changed_for_guard_selection(gs);
+}
+
+/**
+ * Recalculate the list of primary guards (the ones we'd prefer to use) from
+ * the filtered sample and the confirmed list.
+ */
+STATIC void
+entry_guards_update_primary(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  // prevent recursion. Recursion is potentially very bad here.
+  static int running = 0;
+  tor_assert(!running);
+  running = 1;
+
+  const int N_PRIMARY_GUARDS = get_n_primary_guards();
+
+  smartlist_t *new_primary_guards = smartlist_new();
+  smartlist_t *old_primary_guards = smartlist_new();
+  smartlist_add_all(old_primary_guards, gs->primary_entry_guards);
+
+  /* Set this flag now, to prevent the calls below from recursing. */
+  gs->primary_guards_up_to_date = 1;
+
+  /* First, can we fill it up with confirmed guards? */
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (smartlist_len(new_primary_guards) >= N_PRIMARY_GUARDS)
+      break;
+    if (! guard->is_filtered_guard)
+      continue;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Can we keep any older primary guards? First remove all the ones
+   * that we already kept. */
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    if (smartlist_contains(new_primary_guards, guard)) {
+      SMARTLIST_DEL_CURRENT_KEEPORDER(old_primary_guards, guard);
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Now add any that are still good. */
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    if (smartlist_len(new_primary_guards) >= N_PRIMARY_GUARDS)
+      break;
+    if (! guard->is_filtered_guard)
+      continue;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
+    SMARTLIST_DEL_CURRENT_KEEPORDER(old_primary_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Mark the remaining previous primary guards as non-primary */
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    guard->is_primary = 0;
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Finally, fill out the list with sampled guards. */
+  while (smartlist_len(new_primary_guards) < N_PRIMARY_GUARDS) {
+    entry_guard_t *guard = sample_reachable_filtered_entry_guards(gs, NULL,
+                                            SAMPLE_EXCLUDE_CONFIRMED|
+                                            SAMPLE_EXCLUDE_PRIMARY|
+                                            SAMPLE_NO_UPDATE_PRIMARY);
+    if (!guard)
+      break;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
+  }
+
+#if 1
+  /* Debugging. */
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, guard, {
+    tor_assert_nonfatal(
+                   bool_eq(guard->is_primary,
+                           smartlist_contains(new_primary_guards, guard)));
+  });
+#endif /* 1 */
+
+  int any_change = 0;
+  if (smartlist_len(gs->primary_entry_guards) !=
+      smartlist_len(new_primary_guards)) {
+    any_change = 1;
+  } else {
+    SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, g) {
+      if (g != smartlist_get(new_primary_guards, g_sl_idx)) {
+        any_change = 1;
+      }
+    } SMARTLIST_FOREACH_END(g);
+  }
+
+  if (any_change) {
+    log_info(LD_GUARD, "Primary entry guards have changed. "
+             "New primary guard list is: ");
+    int n = smartlist_len(new_primary_guards);
+    SMARTLIST_FOREACH_BEGIN(new_primary_guards, entry_guard_t *, g) {
+      log_info(LD_GUARD, "  %d/%d: %s%s%s",
+               g_sl_idx+1, n, entry_guard_describe(g),
+               g->confirmed_idx >= 0 ? " (confirmed)" : "",
+               g->is_filtered_guard ? "" : " (excluded by filter)");
+    } SMARTLIST_FOREACH_END(g);
+  }
+
+  smartlist_free(old_primary_guards);
+  smartlist_free(gs->primary_entry_guards);
+  gs->primary_entry_guards = new_primary_guards;
+  gs->primary_guards_up_to_date = 1;
+  running = 0;
+}
+
+/**
+ * Return the number of seconds after the last attempt at which we should
+ * retry a guard that has been failing since <b>failing_since</b>.
+ */
+static int
+get_retry_schedule(time_t failing_since, time_t now,
+                   int is_primary)
+{
+  const unsigned SIX_HOURS = 6 * 3600;
+  const unsigned FOUR_DAYS = 4 * 86400;
+  const unsigned SEVEN_DAYS = 7 * 86400;
+
+  time_t tdiff;
+  if (now > failing_since) {
+    tdiff = now - failing_since;
+  } else {
+    tdiff = 0;
+  }
+
+  const struct {
+    time_t maximum; int primary_delay; int nonprimary_delay;
+  } delays[] = {
+    { SIX_HOURS,    10*60,  1*60*60 },
+    { FOUR_DAYS,    90*60,  4*60*60 },
+    { SEVEN_DAYS, 4*60*60, 18*60*60 },
+    { TIME_MAX,   9*60*60, 36*60*60 }
+  };
+
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(delays); ++i) {
+    if (tdiff <= delays[i].maximum) {
+      return is_primary ? delays[i].primary_delay : delays[i].nonprimary_delay;
+    }
+  }
+  /* LCOV_EXCL_START -- can't reach, since delays ends with TIME_MAX. */
+  tor_assert_nonfatal_unreached();
+  return 36*60*60;
+  /* LCOV_EXCL_STOP */
+}
+
+/**
+ * If <b>guard</b> is unreachable, consider whether enough time has passed
+ * to consider it maybe-reachable again.
+ */
+STATIC void
+entry_guard_consider_retry(entry_guard_t *guard)
+{
+  if (guard->is_reachable != GUARD_REACHABLE_NO)
+    return; /* No retry needed. */
+
+  const time_t now = approx_time();
+  const int delay =
+    get_retry_schedule(guard->failing_since, now, guard->is_primary);
+  const time_t last_attempt = guard->last_tried_to_connect;
+
+  if (BUG(last_attempt == 0) ||
+      now >= last_attempt + delay) {
+    /* We should mark this retriable. */
+    char tbuf[ISO_TIME_LEN+1];
+    format_local_iso_time(tbuf, last_attempt);
+    log_info(LD_GUARD, "Marked %s%sguard %s for possible retry, since we "
+             "haven't tried to use it since %s.",
+             guard->is_primary?"primary ":"",
+             guard->confirmed_idx>=0?"confirmed ":"",
+             entry_guard_describe(guard),
+             tbuf);
+
+    guard->is_reachable = GUARD_REACHABLE_MAYBE;
+    if (guard->is_filtered_guard)
+      guard->is_usable_filtered_guard = 1;
+  }
+}
+
+/** Tell the entry guards subsystem that we have confirmed that as of
+ * just now, we're on the internet. */
+void
+entry_guards_note_internet_connectivity(guard_selection_t *gs)
+{
+  gs->last_time_on_internet = approx_time();
+}
+
+/**
+ * Get a guard for use with a circuit.  Prefer to pick a running primary
+ * guard; then a non-pending running filtered confirmed guard; then a
+ * non-pending runnable filtered guard.  Update the
+ * <b>last_tried_to_connect</b> time and the <b>is_pending</b> fields of the
+ * guard as appropriate.  Set <b>state_out</b> to the new guard-state
+ * of the circuit.
+ */
+STATIC entry_guard_t *
+select_entry_guard_for_circuit(guard_selection_t *gs,
+                               guard_usage_t usage,
+                               const entry_guard_restriction_t *rst,
+                               unsigned *state_out)
+{
+  const int need_descriptor = (usage == GUARD_USAGE_TRAFFIC);
+  tor_assert(gs);
+  tor_assert(state_out);
+
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  int num_entry_guards = get_n_primary_guards_to_use(usage);
+  smartlist_t *usable_primary_guards = smartlist_new();
+
+  /* "If any entry in PRIMARY_GUARDS has {is_reachable} status of
+      <maybe> or <yes>, return the first such guard." */
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (guard->is_reachable != GUARD_REACHABLE_NO) {
+      if (need_descriptor && !guard_has_descriptor(guard)) {
+        continue;
+      }
+      *state_out = GUARD_CIRC_STATE_USABLE_ON_COMPLETION;
+      guard->last_tried_to_connect = approx_time();
+      smartlist_add(usable_primary_guards, guard);
+      if (smartlist_len(usable_primary_guards) >= num_entry_guards)
+        break;
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  if (smartlist_len(usable_primary_guards)) {
+    entry_guard_t *guard = smartlist_choose(usable_primary_guards);
+    smartlist_free(usable_primary_guards);
+    log_info(LD_GUARD, "Selected primary guard %s for circuit.",
+             entry_guard_describe(guard));
+    return guard;
+  }
+  smartlist_free(usable_primary_guards);
+
+  /* "Otherwise, if the ordered intersection of {CONFIRMED_GUARDS}
+      and {USABLE_FILTERED_GUARDS} is nonempty, return the first
+      entry in that intersection that has {is_pending} set to
+      false." */
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (guard->is_primary)
+      continue; /* we already considered this one. */
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    entry_guard_consider_retry(guard);
+    if (guard->is_usable_filtered_guard && ! guard->is_pending) {
+      if (need_descriptor && !guard_has_descriptor(guard))
+        continue; /* not a bug */
+      guard->is_pending = 1;
+      guard->last_tried_to_connect = approx_time();
+      *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+      log_info(LD_GUARD, "No primary guards available. Selected confirmed "
+               "guard %s for circuit. Will try other guards before using "
+               "this circuit.",
+               entry_guard_describe(guard));
+      return guard;
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* "Otherwise, if there is no such entry, select a member at
+      random from {USABLE_FILTERED_GUARDS}." */
+  {
+    entry_guard_t *guard;
+    unsigned flags = 0;
+    if (need_descriptor)
+      flags |= SAMPLE_EXCLUDE_NO_DESCRIPTOR;
+    guard = sample_reachable_filtered_entry_guards(gs,
+                                                   rst,
+                                                   SAMPLE_EXCLUDE_CONFIRMED |
+                                                   SAMPLE_EXCLUDE_PRIMARY |
+                                                   SAMPLE_EXCLUDE_PENDING |
+                                                   flags);
+    if (guard == NULL) {
+      log_info(LD_GUARD, "Absolutely no sampled guards were available. "
+               "Marking all guards for retry and starting from top again.");
+      mark_all_guards_maybe_reachable(gs);
+      return NULL;
+    }
+    guard->is_pending = 1;
+    guard->last_tried_to_connect = approx_time();
+    *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+    log_info(LD_GUARD, "No primary or confirmed guards available. Selected "
+             "random guard %s for circuit. Will try other guards before "
+             "using this circuit.",
+             entry_guard_describe(guard));
+    return guard;
+  }
+}
+
+/**
+ * Note that we failed to connect to or build circuits through <b>guard</b>.
+ * Use with a guard returned by select_entry_guard_for_circuit().
+ */
+STATIC void
+entry_guards_note_guard_failure(guard_selection_t *gs,
+                                entry_guard_t *guard)
+{
+  tor_assert(gs);
+
+  guard->is_reachable = GUARD_REACHABLE_NO;
+  guard->is_usable_filtered_guard = 0;
+
+  guard->is_pending = 0;
+  if (guard->failing_since == 0)
+    guard->failing_since = approx_time();
+
+  log_info(LD_GUARD, "Recorded failure for %s%sguard %s",
+           guard->is_primary?"primary ":"",
+           guard->confirmed_idx>=0?"confirmed ":"",
+           entry_guard_describe(guard));
+}
+
+/**
+ * Note that we successfully connected to, and built a circuit through
+ * <b>guard</b>. Given the old guard-state of the circuit in <b>old_state</b>,
+ * return the new guard-state of the circuit.
+ *
+ * Be aware: the circuit is only usable when its guard-state becomes
+ * GUARD_CIRC_STATE_COMPLETE.
+ **/
+STATIC unsigned
+entry_guards_note_guard_success(guard_selection_t *gs,
+                                entry_guard_t *guard,
+                                unsigned old_state)
+{
+  tor_assert(gs);
+
+  /* Save this, since we're about to overwrite it. */
+  const time_t last_time_on_internet = gs->last_time_on_internet;
+  gs->last_time_on_internet = approx_time();
+
+  guard->is_reachable = GUARD_REACHABLE_YES;
+  guard->failing_since = 0;
+  guard->is_pending = 0;
+  if (guard->is_filtered_guard)
+    guard->is_usable_filtered_guard = 1;
+
+  if (guard->confirmed_idx < 0) {
+    make_guard_confirmed(gs, guard);
+    if (!gs->primary_guards_up_to_date)
+      entry_guards_update_primary(gs);
+  }
+
+  unsigned new_state;
+  switch (old_state) {
+    case GUARD_CIRC_STATE_COMPLETE:
+    case GUARD_CIRC_STATE_USABLE_ON_COMPLETION:
+      new_state = GUARD_CIRC_STATE_COMPLETE;
+      break;
+    default:
+      tor_assert_nonfatal_unreached();
+      /* Fall through. */
+    case GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD:
+      if (guard->is_primary) {
+        /* XXXX #20832 -- I don't actually like this logic. It seems to make
+         * us a little more susceptible to evil-ISP attacks.  The mitigations
+         * I'm thinking of, however, aren't local to this point, so I'll leave
+         * it alone. */
+        /* This guard may have become primary by virtue of being confirmed.
+         * If so, the circuit for it is now complete.
+         */
+        new_state = GUARD_CIRC_STATE_COMPLETE;
+      } else {
+        new_state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+      }
+      break;
+  }
+
+  if (! guard->is_primary) {
+    if (last_time_on_internet + get_internet_likely_down_interval()
+        < approx_time()) {
+      mark_primary_guards_maybe_reachable(gs);
+    }
+  }
+
+  log_info(LD_GUARD, "Recorded success for %s%sguard %s",
+           guard->is_primary?"primary ":"",
+           guard->confirmed_idx>=0?"confirmed ":"",
+           entry_guard_describe(guard));
+
+  return new_state;
+}
+
+/**
+ * Helper: Return true iff <b>a</b> has higher priority than <b>b</b>.
+ */
+STATIC int
+entry_guard_has_higher_priority(entry_guard_t *a, entry_guard_t *b)
+{
+  tor_assert(a && b);
+  if (a == b)
+    return 0;
+
+  /* Confirmed is always better than unconfirmed; lower index better
+     than higher */
+  if (a->confirmed_idx < 0) {
+    if (b->confirmed_idx >= 0)
+      return 0;
+  } else {
+    if (b->confirmed_idx < 0)
+      return 1;
+
+    /* Lower confirmed_idx is better than higher. */
+    return (a->confirmed_idx < b->confirmed_idx);
+  }
+
+  /* If we reach this point, both are unconfirmed. If one is pending, it
+   * has higher priority. */
+  if (a->is_pending) {
+    if (! b->is_pending)
+      return 1;
+
+    /* Both are pending: earlier last_tried_connect wins. */
+    return a->last_tried_to_connect < b->last_tried_to_connect;
+  } else {
+    if (b->is_pending)
+      return 0;
+
+    /* Neither is pending: priorities are equal. */
+    return 0;
+  }
+}
+
+/** Release all storage held in <b>restriction</b> */
+STATIC void
+entry_guard_restriction_free_(entry_guard_restriction_t *rst)
+{
+  tor_free(rst);
+}
+
+/**
+ * Release all storage held in <b>state</b>.
+ */
+void
+circuit_guard_state_free_(circuit_guard_state_t *state)
+{
+  if (!state)
+    return;
+  entry_guard_restriction_free(state->restrictions);
+  entry_guard_handle_free(state->guard);
+  tor_free(state);
+}
+
+/** Allocate and return a new circuit_guard_state_t to track the result
+ * of using <b>guard</b> for a given operation. */
+MOCK_IMPL(STATIC circuit_guard_state_t *,
+circuit_guard_state_new,(entry_guard_t *guard, unsigned state,
+                         entry_guard_restriction_t *rst))
+{
+  circuit_guard_state_t *result;
+
+  result = tor_malloc_zero(sizeof(circuit_guard_state_t));
+  result->guard = entry_guard_handle_new(guard);
+  result->state = state;
+  result->state_set_at = approx_time();
+  result->restrictions = rst;
+
+  return result;
+}
+
+/**
+ * Pick a suitable entry guard for a circuit in, and place that guard
+ * in *<b>chosen_node_out</b>. Set *<b>guard_state_out</b> to an opaque
+ * state object that will record whether the circuit is ready to be used
+ * or not. Return 0 on success; on failure, return -1.
+ *
+ * If a restriction is provided in <b>rst</b>, do not return any guards that
+ * violate it, and remember that restriction in <b>guard_state_out</b> for
+ * later use. (Takes ownership of the <b>rst</b> object.)
+ */
+int
+entry_guard_pick_for_circuit(guard_selection_t *gs,
+                             guard_usage_t usage,
+                             entry_guard_restriction_t *rst,
+                             const node_t **chosen_node_out,
+                             circuit_guard_state_t **guard_state_out)
+{
+  tor_assert(gs);
+  tor_assert(chosen_node_out);
+  tor_assert(guard_state_out);
+  *chosen_node_out = NULL;
+  *guard_state_out = NULL;
+
+  unsigned state = 0;
+  entry_guard_t *guard =
+    select_entry_guard_for_circuit(gs, usage, rst, &state);
+  if (! guard)
+    goto fail;
+  if (BUG(state == 0))
+    goto fail;
+  const node_t *node = node_get_by_id(guard->identity);
+  // XXXX #20827 check Ed ID.
+  if (! node)
+    goto fail;
+  if (BUG(usage != GUARD_USAGE_DIRGUARD &&
+          !node_has_preferred_descriptor(node, 1)))
+    goto fail;
+
+  *chosen_node_out = node;
+  *guard_state_out = circuit_guard_state_new(guard, state, rst);
+
+  return 0;
+ fail:
+  entry_guard_restriction_free(rst);
+  return -1;
+}
+
+/**
+ * Called by the circuit building module when a circuit has succeeded: informs
+ * the guards code that the guard in *<b>guard_state_p</b> is working, and
+ * advances the state of the guard module.  On a GUARD_USABLE_NEVER return
+ * value, the circuit is broken and should not be used.  On a GUARD_USABLE_NOW
+ * return value, the circuit is ready to use.  On a GUARD_MAYBE_USABLE_LATER
+ * return value, the circuit should not be used until we find out whether
+ * preferred guards will work for us.
+ */
+guard_usable_t
+entry_guard_succeeded(circuit_guard_state_t **guard_state_p)
+{
+  if (BUG(*guard_state_p == NULL))
+    return GUARD_USABLE_NEVER;
+
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard || BUG(guard->in_selection == NULL))
+    return GUARD_USABLE_NEVER;
+
+  unsigned newstate =
+    entry_guards_note_guard_success(guard->in_selection, guard,
+                                    (*guard_state_p)->state);
+
+  (*guard_state_p)->state = newstate;
+  (*guard_state_p)->state_set_at = approx_time();
+
+  if (newstate == GUARD_CIRC_STATE_COMPLETE) {
+    return GUARD_USABLE_NOW;
+  } else {
+    return GUARD_MAYBE_USABLE_LATER;
+  }
+}
+
+/** Cancel the selection of *<b>guard_state_p</b> without declaring
+ * success or failure. It is safe to call this function if success or
+ * failure _has_ already been declared. */
+void
+entry_guard_cancel(circuit_guard_state_t **guard_state_p)
+{
+  if (BUG(*guard_state_p == NULL))
+    return;
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard)
+    return;
+
+  /* XXXX prop271 -- last_tried_to_connect_at will be erroneous here, but this
+   * function will only get called in "bug" cases anyway. */
+  guard->is_pending = 0;
+  circuit_guard_state_free(*guard_state_p);
+  *guard_state_p = NULL;
+}
+
+/**
+ * Called by the circuit building module when a circuit has failed:
+ * informs the guards code that the guard in *<b>guard_state_p</b> is
+ * not working, and advances the state of the guard module.
+ */
+void
+entry_guard_failed(circuit_guard_state_t **guard_state_p)
+{
+  if (BUG(*guard_state_p == NULL))
+    return;
+
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard || BUG(guard->in_selection == NULL))
+    return;
+
+  entry_guards_note_guard_failure(guard->in_selection, guard);
+
+  (*guard_state_p)->state = GUARD_CIRC_STATE_DEAD;
+  (*guard_state_p)->state_set_at = approx_time();
+}
+
+/**
+ * Run the entry_guard_failed() function on every circuit that is
+ * pending on <b>chan</b>.
+ */
+void
+entry_guard_chan_failed(channel_t *chan)
+{
+  if (!chan)
+    return;
+
+  smartlist_t *pending = smartlist_new();
+  circuit_get_all_pending_on_channel(pending, chan);
+  SMARTLIST_FOREACH_BEGIN(pending, circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+
+    origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ);
+    if (origin_circ->guard_state) {
+      /* We might have no guard state if we didn't use a guard on this
+       * circuit (eg it's for a fallback directory). */
+      entry_guard_failed(&origin_circ->guard_state);
+    }
+  } SMARTLIST_FOREACH_END(circ);
+  smartlist_free(pending);
+}
+
+/**
+ * Return true iff every primary guard in <b>gs</b> is believed to
+ * be unreachable.
+ */
+STATIC int
+entry_guards_all_primary_guards_are_down(guard_selection_t *gs)
+{
+  tor_assert(gs);
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (guard->is_reachable != GUARD_REACHABLE_NO)
+      return 0;
+  } SMARTLIST_FOREACH_END(guard);
+  return 1;
+}
+
+/** Wrapper for entry_guard_has_higher_priority that compares the
+ * guard-priorities of a pair of circuits. Return 1 if <b>a</b> has higher
+ * priority than <b>b</b>.
+ *
+ * If a restriction is provided in <b>rst</b>, then do not consider
+ * <b>a</b> to have higher priority if it violates the restriction.
+ */
+static int
+circ_state_has_higher_priority(origin_circuit_t *a,
+                               const entry_guard_restriction_t *rst,
+                               origin_circuit_t *b)
+{
+  circuit_guard_state_t *state_a = origin_circuit_get_guard_state(a);
+  circuit_guard_state_t *state_b = origin_circuit_get_guard_state(b);
+
+  tor_assert(state_a);
+  tor_assert(state_b);
+
+  entry_guard_t *guard_a = entry_guard_handle_get(state_a->guard);
+  entry_guard_t *guard_b = entry_guard_handle_get(state_b->guard);
+
+  if (! guard_a) {
+    /* Unknown guard -- never higher priority. */
+    return 0;
+  } else if (! guard_b) {
+    /* Known guard -- higher priority than any unknown guard. */
+    return 1;
+  } else  if (! entry_guard_obeys_restriction(guard_a, rst)) {
+    /* Restriction violated; guard_a cannot have higher priority. */
+    return 0;
+  } else {
+    /* Both known -- compare.*/
+    return entry_guard_has_higher_priority(guard_a, guard_b);
+  }
+}
+
+/**
+ * Look at all of the origin_circuit_t * objects in <b>all_circuits_in</b>,
+ * and see if any of them that were previously not ready to use for
+ * guard-related reasons are now ready to use. Place those circuits
+ * in <b>newly_complete_out</b>, and mark them COMPLETE.
+ *
+ * Return 1 if we upgraded any circuits, and 0 otherwise.
+ */
+int
+entry_guards_upgrade_waiting_circuits(guard_selection_t *gs,
+                                      const smartlist_t *all_circuits_in,
+                                      smartlist_t *newly_complete_out)
+{
+  tor_assert(gs);
+  tor_assert(all_circuits_in);
+  tor_assert(newly_complete_out);
+
+  if (! entry_guards_all_primary_guards_are_down(gs)) {
+    /* We only upgrade a waiting circuit if the primary guards are all
+     * down. */
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits, "
+              "but not all primary guards were definitely down.");
+    return 0;
+  }
+
+  int n_waiting = 0;
+  int n_complete = 0;
+  int n_complete_blocking = 0;
+  origin_circuit_t *best_waiting_circuit = NULL;
+  smartlist_t *all_circuits = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(all_circuits_in, origin_circuit_t *, circ) {
+    // We filter out circuits that aren't ours, or which we can't
+    // reason about.
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (state == NULL)
+      continue;
+    entry_guard_t *guard = entry_guard_handle_get(state->guard);
+    if (!guard || guard->in_selection != gs)
+      continue;
+
+    smartlist_add(all_circuits, circ);
+  } SMARTLIST_FOREACH_END(circ);
+
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+
+    if (state->state == GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD) {
+      ++n_waiting;
+      if (! best_waiting_circuit ||
+          circ_state_has_higher_priority(circ, NULL, best_waiting_circuit)) {
+        best_waiting_circuit = circ;
+      }
+    }
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (! best_waiting_circuit) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits, "
+              "but didn't find any.");
+    goto no_change;
+  }
+
+  /* We'll need to keep track of what restrictions were used when picking this
+   * circuit, so that we don't allow any circuit without those restrictions to
+   * block it. */
+  const entry_guard_restriction_t *rst_on_best_waiting =
+    origin_circuit_get_guard_state(best_waiting_circuit)->restrictions;
+
+  /* First look at the complete circuits: Do any block this circuit? */
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    /* "C2 "blocks" C1 if:
+        * C2 obeys all the restrictions that C1 had to obey, AND
+        * C2 has higher priority than C1, AND
+        * Either C2 is <complete>, or C2 is <waiting_for_better_guard>,
+          or C2 has been <usable_if_no_better_guard> for no more than
+          {NONPRIMARY_GUARD_CONNECT_TIMEOUT} seconds."
+    */
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if BUG((state == NULL))
+      continue;
+    if (state->state != GUARD_CIRC_STATE_COMPLETE)
+      continue;
+    ++n_complete;
+    if (circ_state_has_higher_priority(circ, rst_on_best_waiting,
+                                       best_waiting_circuit))
+      ++n_complete_blocking;
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (n_complete_blocking) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+              "%d complete and %d guard-stalled. At least one complete "
+              "circuit had higher priority, so not upgrading.",
+              n_complete, n_waiting);
+    goto no_change;
+  }
+
+  /* " * If any circuit C1 is <waiting_for_better_guard>, AND:
+          * All primary guards have reachable status of <no>.
+          * There is no circuit C2 that "blocks" C1.
+         Then, upgrade C1 to <complete>.""
+  */
+  int n_blockers_found = 0;
+  const time_t state_set_at_cutoff =
+    approx_time() - get_nonprimary_guard_connect_timeout();
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+    if (state->state != GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD)
+      continue;
+    if (state->state_set_at <= state_set_at_cutoff)
+      continue;
+    if (circ_state_has_higher_priority(circ, rst_on_best_waiting,
+                                       best_waiting_circuit))
+      ++n_blockers_found;
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (n_blockers_found) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+              "%d guard-stalled, but %d pending circuit(s) had higher "
+              "guard priority, so not upgrading.",
+              n_waiting, n_blockers_found);
+    goto no_change;
+  }
+
+  /* Okay. We have a best waiting circuit, and we aren't waiting for
+     anything better.  Add all circuits with that priority to the
+     list, and call them COMPLETE. */
+  int n_succeeded = 0;
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+    if (circ != best_waiting_circuit && rst_on_best_waiting) {
+      /* Can't upgrade other circ with same priority as best; might
+         be blocked. */
+      continue;
+    }
+    if (state->state != GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD)
+      continue;
+    if (circ_state_has_higher_priority(best_waiting_circuit, NULL, circ))
+      continue;
+
+    state->state = GUARD_CIRC_STATE_COMPLETE;
+    state->state_set_at = approx_time();
+    smartlist_add(newly_complete_out, circ);
+    ++n_succeeded;
+  } SMARTLIST_FOREACH_END(circ);
+
+  log_info(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+           "%d guard-stalled, %d complete. %d of the guard-stalled "
+           "circuit(s) had high enough priority to upgrade.",
+           n_waiting, n_complete, n_succeeded);
+
+  tor_assert_nonfatal(n_succeeded >= 1);
+  smartlist_free(all_circuits);
+  return 1;
+
+ no_change:
+  smartlist_free(all_circuits);
+  return 0;
+}
+
+/**
+ * Return true iff the circuit whose state is <b>guard_state</b> should
+ * expire.
+ */
+int
+entry_guard_state_should_expire(circuit_guard_state_t *guard_state)
+{
+  if (guard_state == NULL)
+    return 0;
+  const time_t expire_if_waiting_since =
+    approx_time() - get_nonprimary_guard_idle_timeout();
+  return (guard_state->state == GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD
+          && guard_state->state_set_at < expire_if_waiting_since);
+}
+
+/**
+ * Update all derived pieces of the guard selection state in <b>gs</b>.
+ * Return true iff we should stop using all previously generated circuits.
+ */
+int
+entry_guards_update_all(guard_selection_t *gs)
+{
+  sampled_guards_update_from_consensus(gs);
+  entry_guards_update_filtered_sets(gs);
+  entry_guards_update_confirmed(gs);
+  entry_guards_update_primary(gs);
+  return 0;
+}
+
+/**
+ * Return a newly allocated string for encoding the persistent parts of
+ * <b>guard</b> to the state file.
+ */
+STATIC char *
+entry_guard_encode_for_state(entry_guard_t *guard)
+{
+  /*
+   * The meta-format we use is K=V K=V K=V... where K can be any
+   * characters excepts space and =, and V can be any characters except
+   * space.  The order of entries is not allowed to matter.
+   * Unrecognized K=V entries are persisted; recognized but erroneous
+   * entries are corrected.
+   */
+
+  smartlist_t *result = smartlist_new();
+  char tbuf[ISO_TIME_LEN+1];
+
+  tor_assert(guard);
+
+  smartlist_add_asprintf(result, "in=%s", guard->selection_name);
+  smartlist_add_asprintf(result, "rsa_id=%s",
+                         hex_str(guard->identity, DIGEST_LEN));
+  if (guard->bridge_addr) {
+    smartlist_add_asprintf(result, "bridge_addr=%s:%d",
+                           fmt_and_decorate_addr(&guard->bridge_addr->addr),
+                           guard->bridge_addr->port);
+  }
+  if (strlen(guard->nickname) && is_legal_nickname(guard->nickname)) {
+    smartlist_add_asprintf(result, "nickname=%s", guard->nickname);
+  }
+
+  format_iso_time_nospace(tbuf, guard->sampled_on_date);
+  smartlist_add_asprintf(result, "sampled_on=%s", tbuf);
+
+  if (guard->sampled_by_version) {
+    smartlist_add_asprintf(result, "sampled_by=%s",
+                           guard->sampled_by_version);
+  }
+
+  if (guard->unlisted_since_date > 0) {
+    format_iso_time_nospace(tbuf, guard->unlisted_since_date);
+    smartlist_add_asprintf(result, "unlisted_since=%s", tbuf);
+  }
+
+  smartlist_add_asprintf(result, "listed=%d",
+                         (int)guard->currently_listed);
+
+  if (guard->confirmed_idx >= 0) {
+    format_iso_time_nospace(tbuf, guard->confirmed_on_date);
+    smartlist_add_asprintf(result, "confirmed_on=%s", tbuf);
+
+    smartlist_add_asprintf(result, "confirmed_idx=%d", guard->confirmed_idx);
+  }
+
+  const double EPSILON = 1.0e-6;
+
+  /* Make a copy of the pathbias object, since we will want to update
+     some of them */
+  guard_pathbias_t *pb = tor_memdup(&guard->pb, sizeof(*pb));
+  pb->use_successes = pathbias_get_use_success_count(guard);
+  pb->successful_circuits_closed = pathbias_get_close_success_count(guard);
+
+  #define PB_FIELD(field) do {                                          \
+      if (pb->field >= EPSILON) {                                       \
+        smartlist_add_asprintf(result, "pb_" #field "=%f", pb->field);  \
+      }                                                                 \
+    } while (0)
+  PB_FIELD(use_attempts);
+  PB_FIELD(use_successes);
+  PB_FIELD(circ_attempts);
+  PB_FIELD(circ_successes);
+  PB_FIELD(successful_circuits_closed);
+  PB_FIELD(collapsed_circuits);
+  PB_FIELD(unusable_circuits);
+  PB_FIELD(timeouts);
+  tor_free(pb);
+#undef PB_FIELD
+
+  if (guard->extra_state_fields)
+    smartlist_add_strdup(result, guard->extra_state_fields);
+
+  char *joined = smartlist_join_strings(result, " ", 0, NULL);
+  SMARTLIST_FOREACH(result, char *, cp, tor_free(cp));
+  smartlist_free(result);
+
+  return joined;
+}
+
+/**
+ * Given a string generated by entry_guard_encode_for_state(), parse it
+ * (if possible) and return an entry_guard_t object for it.  Return NULL
+ * on complete failure.
+ */
+STATIC entry_guard_t *
+entry_guard_parse_from_state(const char *s)
+{
+  /* Unrecognized entries get put in here. */
+  smartlist_t *extra = smartlist_new();
+
+  /* These fields get parsed from the string. */
+  char *in = NULL;
+  char *rsa_id = NULL;
+  char *nickname = NULL;
+  char *sampled_on = NULL;
+  char *sampled_by = NULL;
+  char *unlisted_since = NULL;
+  char *listed  = NULL;
+  char *confirmed_on = NULL;
+  char *confirmed_idx = NULL;
+  char *bridge_addr = NULL;
+
+  // pathbias
+  char *pb_use_attempts = NULL;
+  char *pb_use_successes = NULL;
+  char *pb_circ_attempts = NULL;
+  char *pb_circ_successes = NULL;
+  char *pb_successful_circuits_closed = NULL;
+  char *pb_collapsed_circuits = NULL;
+  char *pb_unusable_circuits = NULL;
+  char *pb_timeouts = NULL;
+
+  /* Split up the entries.  Put the ones we know about in strings and the
+   * rest in "extra". */
+  {
+    smartlist_t *entries = smartlist_new();
+
+    strmap_t *vals = strmap_new(); // Maps keyword to location
+#define FIELD(f) \
+    strmap_set(vals, #f, &f);
+    FIELD(in);
+    FIELD(rsa_id);
+    FIELD(nickname);
+    FIELD(sampled_on);
+    FIELD(sampled_by);
+    FIELD(unlisted_since);
+    FIELD(listed);
+    FIELD(confirmed_on);
+    FIELD(confirmed_idx);
+    FIELD(bridge_addr);
+    FIELD(pb_use_attempts);
+    FIELD(pb_use_successes);
+    FIELD(pb_circ_attempts);
+    FIELD(pb_circ_successes);
+    FIELD(pb_successful_circuits_closed);
+    FIELD(pb_collapsed_circuits);
+    FIELD(pb_unusable_circuits);
+    FIELD(pb_timeouts);
+#undef FIELD
+
+    smartlist_split_string(entries, s, " ",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+    SMARTLIST_FOREACH_BEGIN(entries, char *, entry) {
+      const char *eq = strchr(entry, '=');
+      if (!eq) {
+        smartlist_add(extra, entry);
+        continue;
+      }
+      char *key = tor_strndup(entry, eq-entry);
+      char **target = strmap_get(vals, key);
+      if (target == NULL || *target != NULL) {
+        /* unrecognized or already set */
+        smartlist_add(extra, entry);
+        tor_free(key);
+        continue;
+      }
+
+      *target = tor_strdup(eq+1);
+      tor_free(key);
+      tor_free(entry);
+    } SMARTLIST_FOREACH_END(entry);
+
+    smartlist_free(entries);
+    strmap_free(vals, NULL);
+  }
+
+  entry_guard_t *guard = tor_malloc_zero(sizeof(entry_guard_t));
+  guard->is_persistent = 1;
+
+  if (in == NULL) {
+    log_warn(LD_CIRC, "Guard missing 'in' field");
+    goto err;
+  }
+
+  guard->selection_name = in;
+  in = NULL;
+
+  if (rsa_id == NULL) {
+    log_warn(LD_CIRC, "Guard missing RSA ID field");
+    goto err;
+  }
+
+  /* Process the identity and nickname. */
+  if (base16_decode(guard->identity, sizeof(guard->identity),
+                    rsa_id, strlen(rsa_id)) != DIGEST_LEN) {
+    log_warn(LD_CIRC, "Unable to decode guard identity %s", escaped(rsa_id));
+    goto err;
+  }
+
+  if (nickname) {
+    strlcpy(guard->nickname, nickname, sizeof(guard->nickname));
+  } else {
+    guard->nickname[0]='$';
+    base16_encode(guard->nickname+1, sizeof(guard->nickname)-1,
+                  guard->identity, DIGEST_LEN);
+  }
+
+  if (bridge_addr) {
+    tor_addr_port_t res;
+    memset(&res, 0, sizeof(res));
+    int r = tor_addr_port_parse(LOG_WARN, bridge_addr,
+                                &res.addr, &res.port, -1);
+    if (r == 0)
+      guard->bridge_addr = tor_memdup(&res, sizeof(res));
+    /* On error, we already warned. */
+  }
+
+  /* Process the various time fields. */
+
+#define HANDLE_TIME(field) do {                                 \
+    if (field) {                                                \
+      int r = parse_iso_time_nospace(field, &field ## _time);   \
+      if (r < 0) {                                              \
+        log_warn(LD_CIRC, "Unable to parse %s %s from guard",   \
+                 #field, escaped(field));                       \
+        field##_time = -1;                                      \
+      }                                                         \
+    }                                                           \
+  } while (0)
+
+  time_t sampled_on_time = 0;
+  time_t unlisted_since_time = 0;
+  time_t confirmed_on_time = 0;
+
+  HANDLE_TIME(sampled_on);
+  HANDLE_TIME(unlisted_since);
+  HANDLE_TIME(confirmed_on);
+
+  if (sampled_on_time <= 0)
+    sampled_on_time = approx_time();
+  if (unlisted_since_time < 0)
+    unlisted_since_time = 0;
+  if (confirmed_on_time < 0)
+    confirmed_on_time = 0;
+
+  #undef HANDLE_TIME
+
+  guard->sampled_on_date = sampled_on_time;
+  guard->unlisted_since_date = unlisted_since_time;
+  guard->confirmed_on_date = confirmed_on_time;
+
+  /* Take sampled_by_version verbatim. */
+  guard->sampled_by_version = sampled_by;
+  sampled_by = NULL; /* prevent free */
+
+  /* Listed is a boolean */
+  if (listed && strcmp(listed, "0"))
+    guard->currently_listed = 1;
+
+  /* The index is a nonnegative integer. */
+  guard->confirmed_idx = -1;
+  if (confirmed_idx) {
+    int ok=1;
+    long idx = tor_parse_long(confirmed_idx, 10, 0, INT_MAX, &ok, NULL);
+    if (! ok) {
+      log_warn(LD_GUARD, "Guard has invalid confirmed_idx %s",
+               escaped(confirmed_idx));
+    } else {
+      guard->confirmed_idx = (int)idx;
+    }
+  }
+
+  /* Anything we didn't recognize gets crammed together */
+  if (smartlist_len(extra) > 0) {
+    guard->extra_state_fields = smartlist_join_strings(extra, " ", 0, NULL);
+  }
+
+  /* initialize non-persistent fields */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+
+#define PB_FIELD(field)                                                 \
+  do {                                                                  \
+    if (pb_ ## field) {                                                 \
+      int ok = 1;                                                       \
+      double r = tor_parse_double(pb_ ## field, 0.0, 1e9, &ok, NULL);   \
+      if (! ok) {                                                       \
+        log_warn(LD_CIRC, "Guard has invalid pb_%s %s",                 \
+                 #field, pb_ ## field);                                 \
+      } else {                                                          \
+        guard->pb.field = r;                                            \
+      }                                                                 \
+    }                                                                   \
+  } while (0)
+  PB_FIELD(use_attempts);
+  PB_FIELD(use_successes);
+  PB_FIELD(circ_attempts);
+  PB_FIELD(circ_successes);
+  PB_FIELD(successful_circuits_closed);
+  PB_FIELD(collapsed_circuits);
+  PB_FIELD(unusable_circuits);
+  PB_FIELD(timeouts);
+#undef PB_FIELD
+
+  pathbias_check_use_success_count(guard);
+  pathbias_check_close_success_count(guard);
+
+  /* We update everything on this guard later, after we've parsed
+   * everything.  */
+
+  goto done;
+
+ err:
+  // only consider it an error if the guard state was totally unparseable.
+  entry_guard_free(guard);
+  guard = NULL;
+
+ done:
+  tor_free(in);
+  tor_free(rsa_id);
+  tor_free(nickname);
+  tor_free(sampled_on);
+  tor_free(sampled_by);
+  tor_free(unlisted_since);
+  tor_free(listed);
+  tor_free(confirmed_on);
+  tor_free(confirmed_idx);
+  tor_free(bridge_addr);
+  tor_free(pb_use_attempts);
+  tor_free(pb_use_successes);
+  tor_free(pb_circ_attempts);
+  tor_free(pb_circ_successes);
+  tor_free(pb_successful_circuits_closed);
+  tor_free(pb_collapsed_circuits);
+  tor_free(pb_unusable_circuits);
+  tor_free(pb_timeouts);
+
+  SMARTLIST_FOREACH(extra, char *, cp, tor_free(cp));
+  smartlist_free(extra);
+
+  return guard;
+}
+
+/**
+ * Replace the Guards entries in <b>state</b> with a list of all our sampled
+ * guards.
+ */
+static void
+entry_guards_update_guards_in_state(or_state_t *state)
+{
+  if (!guard_contexts)
+    return;
+  config_line_t *lines = NULL;
+  config_line_t **nextline = &lines;
+
+  SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+      if (guard->is_persistent == 0)
+        continue;
+      *nextline = tor_malloc_zero(sizeof(config_line_t));
+      (*nextline)->key = tor_strdup("Guard");
+      (*nextline)->value = entry_guard_encode_for_state(guard);
+      nextline = &(*nextline)->next;
+    } SMARTLIST_FOREACH_END(guard);
+  } SMARTLIST_FOREACH_END(gs);
+
+  config_free_lines(state->Guard);
+  state->Guard = lines;
+}
+
+/**
+ * Replace our sampled guards from the Guards entries in <b>state</b>. Return 0
+ * on success, -1 on failure. (If <b>set</b> is true, replace nothing -- only
+ * check whether replacing would work.)
+ */
+static int
+entry_guards_load_guards_from_state(or_state_t *state, int set)
+{
+  const config_line_t *line = state->Guard;
+  int n_errors = 0;
+
+  if (!guard_contexts)
+    guard_contexts = smartlist_new();
+
+  /* Wipe all our existing guard info. (we shouldn't have any, but
+   * let's be safe.) */
+  if (set) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      guard_selection_free(gs);
+      if (curr_guard_context == gs)
+        curr_guard_context = NULL;
+      SMARTLIST_DEL_CURRENT(guard_contexts, gs);
+    } SMARTLIST_FOREACH_END(gs);
+  }
+
+  for ( ; line != NULL; line = line->next) {
+    entry_guard_t *guard = entry_guard_parse_from_state(line->value);
+    if (guard == NULL) {
+      ++n_errors;
+      continue;
+    }
+    tor_assert(guard->selection_name);
+    if (!strcmp(guard->selection_name, "legacy")) {
+      ++n_errors;
+      entry_guard_free(guard);
+      continue;
+    }
+
+    if (set) {
+      guard_selection_t *gs;
+      gs = get_guard_selection_by_name(guard->selection_name,
+                                       GS_TYPE_INFER, 1);
+      tor_assert(gs);
+      smartlist_add(gs->sampled_entry_guards, guard);
+      guard->in_selection = gs;
+    } else {
+      entry_guard_free(guard);
+    }
+  }
+
+  if (set) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      entry_guards_update_all(gs);
+    } SMARTLIST_FOREACH_END(gs);
+  }
+  return n_errors ? -1 : 0;
+}
+
+/** If <b>digest</b> matches the identity of any node in the
+ * entry_guards list for the provided guard selection state,
+ return that node. Else return NULL. */
+entry_guard_t *
+entry_guard_get_by_id_digest_for_guard_selection(guard_selection_t *gs,
+                                                 const char *digest)
+{
+  return get_sampled_guard_with_id(gs, (const uint8_t*)digest);
+}
+
+/** Return the node_t associated with a single entry_guard_t. May
+ * return NULL if the guard is not currently in the consensus. */
+const node_t *
+entry_guard_find_node(const entry_guard_t *guard)
+{
+  tor_assert(guard);
+  return node_get_by_id(guard->identity);
+}
+
+/** If <b>digest</b> matches the identity of any node in the
+ * entry_guards list for the default guard selection state,
+ return that node. Else return NULL. */
+entry_guard_t *
+entry_guard_get_by_id_digest(const char *digest)
+{
+  return entry_guard_get_by_id_digest_for_guard_selection(
+      get_guard_selection_info(), digest);
+}
+
+/** We are about to connect to bridge with identity <b>digest</b> to fetch its
+ *  descriptor. Create a new guard state for this connection and return it. */
+circuit_guard_state_t *
+get_guard_state_for_bridge_desc_fetch(const char *digest)
+{
+  circuit_guard_state_t *guard_state = NULL;
+  entry_guard_t *guard = NULL;
+
+  guard = entry_guard_get_by_id_digest_for_guard_selection(
+                                    get_guard_selection_info(), digest);
+  if (!guard) {
+    return NULL;
+  }
+
+  /* Update the guard last_tried_to_connect time since it's checked by the
+   * guard susbsystem. */
+  guard->last_tried_to_connect = approx_time();
+
+  /* Create the guard state */
+  guard_state = circuit_guard_state_new(guard,
+                                        GUARD_CIRC_STATE_USABLE_ON_COMPLETION,
+                                        NULL);
+
+  return guard_state;
+}
+
+/** Release all storage held by <b>e</b>. */
+STATIC void
+entry_guard_free_(entry_guard_t *e)
+{
+  if (!e)
+    return;
+  entry_guard_handles_clear(e);
+  tor_free(e->sampled_by_version);
+  tor_free(e->extra_state_fields);
+  tor_free(e->selection_name);
+  tor_free(e->bridge_addr);
+  tor_free(e);
+}
+
+/** Return 0 if we're fine adding arbitrary routers out of the
+ * directory to our entry guard list, or return 1 if we have a
+ * list already and we must stick to it.
+ */
+int
+entry_list_is_constrained(const or_options_t *options)
+{
+  // XXXX #21425 look at the current selection.
+  if (options->EntryNodes)
+    return 1;
+  if (options->UseBridges)
+    return 1;
+  return 0;
+}
+
+/** Return the number of bridges that have descriptors that are marked with
+ * purpose 'bridge' and are running. If use_maybe_reachable is
+ * true, include bridges that might be reachable in the count.
+ * Otherwise, if it is false, only include bridges that have recently been
+ * found running in the count.
+ *
+ * We use this function to decide if we're ready to start building
+ * circuits through our bridges, or if we need to wait until the
+ * directory "server/authority" requests finish. */
+MOCK_IMPL(int,
+num_bridges_usable,(int use_maybe_reachable))
+{
+  int n_options = 0;
+
+  if (BUG(!get_options()->UseBridges)) {
+    return 0;
+  }
+  guard_selection_t *gs  = get_guard_selection_info();
+  if (BUG(gs->type != GS_TYPE_BRIDGE)) {
+    return 0;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    /* Definitely not usable */
+    if (guard->is_reachable == GUARD_REACHABLE_NO)
+      continue;
+    /* If we want to be really sure the bridges will work, skip maybes */
+    if (!use_maybe_reachable && guard->is_reachable == GUARD_REACHABLE_MAYBE)
+      continue;
+    if (tor_digest_is_zero(guard->identity))
+      continue;
+    const node_t *node = node_get_by_id(guard->identity);
+    if (node && node->ri)
+      ++n_options;
+  } SMARTLIST_FOREACH_END(guard);
+
+  return n_options;
+}
+
+/** Check the pathbias use success count of <b>node</b> and disable it if it
+ *  goes over our thresholds. */
+static void
+pathbias_check_use_success_count(entry_guard_t *node)
+{
+  const or_options_t *options = get_options();
+  const double EPSILON = 1.0e-9;
+
+  /* Note: We rely on the < comparison here to allow us to set a 0
+   * rate and disable the feature entirely. If refactoring, don't
+   * change to <= */
+  if (node->pb.use_attempts > EPSILON &&
+      pathbias_get_use_success_count(node)/node->pb.use_attempts
+      < pathbias_get_extreme_use_rate(options) &&
+      pathbias_get_dropguards(options)) {
+    node->pb.path_bias_disabled = 1;
+    log_info(LD_GENERAL,
+             "Path use bias is too high (%f/%f); disabling node %s",
+             node->pb.circ_successes, node->pb.circ_attempts,
+             node->nickname);
+  }
+}
+
+/** Check the pathbias close count of <b>node</b> and disable it if it goes
+ *  over our thresholds. */
+static void
+pathbias_check_close_success_count(entry_guard_t *node)
+{
+  const or_options_t *options = get_options();
+  const double EPSILON = 1.0e-9;
+
+  /* Note: We rely on the < comparison here to allow us to set a 0
+   * rate and disable the feature entirely. If refactoring, don't
+   * change to <= */
+  if (node->pb.circ_attempts > EPSILON &&
+      pathbias_get_close_success_count(node)/node->pb.circ_attempts
+      < pathbias_get_extreme_rate(options) &&
+      pathbias_get_dropguards(options)) {
+    node->pb.path_bias_disabled = 1;
+    log_info(LD_GENERAL,
+             "Path bias is too high (%f/%f); disabling node %s",
+             node->pb.circ_successes, node->pb.circ_attempts,
+             node->nickname);
+  }
+}
+
+/** Parse <b>state</b> and learn about the entry guards it describes.
+ * If <b>set</b> is true, and there are no errors, replace the guard
+ * list in the default guard selection context with what we find.
+ * On success, return 0. On failure, alloc into *<b>msg</b> a string
+ * describing the error, and return -1.
+ */
+int
+entry_guards_parse_state(or_state_t *state, int set, char **msg)
+{
+  entry_guards_dirty = 0;
+  int r1 = entry_guards_load_guards_from_state(state, set);
+  entry_guards_dirty = 0;
+
+  if (r1 < 0) {
+    if (msg && *msg == NULL) {
+      *msg = tor_strdup("parsing error");
+    }
+    return -1;
+  }
+  return 0;
+}
+
+/** How long will we let a change in our guard nodes stay un-saved
+ * when we are trying to avoid disk writes? */
+#define SLOW_GUARD_STATE_FLUSH_TIME 600
+/** How long will we let a change in our guard nodes stay un-saved
+ * when we are not trying to avoid disk writes? */
+#define FAST_GUARD_STATE_FLUSH_TIME 30
+
+/** Our list of entry guards has changed for a particular guard selection
+ * context, or some element of one of our entry guards has changed for one.
+ * Write the changes to disk within the next few minutes.
+ */
+void
+entry_guards_changed_for_guard_selection(guard_selection_t *gs)
+{
+  time_t when;
+
+  tor_assert(gs != NULL);
+
+  entry_guards_dirty = 1;
+
+  if (get_options()->AvoidDiskWrites)
+    when = time(NULL) + SLOW_GUARD_STATE_FLUSH_TIME;
+  else
+    when = time(NULL) + FAST_GUARD_STATE_FLUSH_TIME;
+
+  /* or_state_save() will call entry_guards_update_state() and
+     entry_guards_update_guards_in_state()
+  */
+  or_state_mark_dirty(get_or_state(), when);
+}
+
+/** Our list of entry guards has changed for the default guard selection
+ * context, or some element of one of our entry guards has changed. Write
+ * the changes to disk within the next few minutes.
+ */
+void
+entry_guards_changed(void)
+{
+  entry_guards_changed_for_guard_selection(get_guard_selection_info());
+}
+
+/** If the entry guard info has not changed, do nothing and return.
+ * Otherwise, free the EntryGuards piece of <b>state</b> and create
+ * a new one out of the global entry_guards list, and then mark
+ * <b>state</b> dirty so it will get saved to disk.
+ */
+void
+entry_guards_update_state(or_state_t *state)
+{
+  entry_guards_dirty = 0;
+
+  // Handles all guard info.
+  entry_guards_update_guards_in_state(state);
+
+  entry_guards_dirty = 0;
+
+  if (!get_options()->AvoidDiskWrites)
+    or_state_mark_dirty(get_or_state(), 0);
+  entry_guards_dirty = 0;
+}
+
+/** Return true iff the circuit's guard can succeed that is can be used. */
+int
+entry_guard_could_succeed(const circuit_guard_state_t *guard_state)
+{
+  if (!guard_state) {
+    return 0;
+  }
+
+  entry_guard_t *guard = entry_guard_handle_get(guard_state->guard);
+  if (!guard || BUG(guard->in_selection == NULL)) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * Format a single entry guard in the format expected by the controller.
+ * Return a newly allocated string.
+ */
+STATIC char *
+getinfo_helper_format_single_entry_guard(const entry_guard_t *e)
+{
+  const char *status = NULL;
+  time_t when = 0;
+  const node_t *node;
+  char tbuf[ISO_TIME_LEN+1];
+  char nbuf[MAX_VERBOSE_NICKNAME_LEN+1];
+
+  /* This is going to be a bit tricky, since the status
+   * codes weren't really intended for prop271 guards.
+   *
+   * XXXX use a more appropriate format for exporting this information
+   */
+  if (e->confirmed_idx < 0) {
+    status = "never-connected";
+  } else if (! e->currently_listed) {
+    when = e->unlisted_since_date;
+    status = "unusable";
+  } else if (! e->is_filtered_guard) {
+    status = "unusable";
+  } else if (e->is_reachable == GUARD_REACHABLE_NO) {
+    when = e->failing_since;
+    status = "down";
+  } else {
+    status = "up";
+  }
+
+  node = entry_guard_find_node(e);
+  if (node) {
+    node_get_verbose_nickname(node, nbuf);
+  } else {
+    nbuf[0] = '$';
+    base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN);
+    /* e->nickname field is not very reliable if we don't know about
+     * this router any longer; don't include it. */
+  }
+
+  char *result = NULL;
+  if (when) {
+    format_iso_time(tbuf, when);
+    tor_asprintf(&result, "%s %s %s\n", nbuf, status, tbuf);
+  } else {
+    tor_asprintf(&result, "%s %s\n", nbuf, status);
+  }
+  return result;
+}
+
+/** If <b>question</b> is the string "entry-guards", then dump
+ * to *<b>answer</b> a newly allocated string describing all of
+ * the nodes in the global entry_guards list. See control-spec.txt
+ * for details.
+ * For backward compatibility, we also handle the string "helper-nodes".
+ *
+ * XXX this should be totally redesigned after prop 271 too, and that's
+ * going to take some control spec work.
+ * */
+int
+getinfo_helper_entry_guards(control_connection_t *conn,
+                            const char *question, char **answer,
+                            const char **errmsg)
+{
+  guard_selection_t *gs = get_guard_selection_info();
+
+  tor_assert(gs != NULL);
+
+  (void) conn;
+  (void) errmsg;
+
+  if (!strcmp(question,"entry-guards") ||
+      !strcmp(question,"helper-nodes")) {
+    const smartlist_t *guards;
+    guards = gs->sampled_entry_guards;
+
+    smartlist_t *sl = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(guards, const entry_guard_t *, e) {
+      char *cp = getinfo_helper_format_single_entry_guard(e);
+      smartlist_add(sl, cp);
+    } SMARTLIST_FOREACH_END(e);
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  }
+  return 0;
+}
+
+/* Given the original bandwidth of a guard and its guardfraction,
+ * calculate how much bandwidth the guard should have as a guard and
+ * as a non-guard.
+ *
+ * Quoting from proposal236:
+ *
+ *   Let Wpf denote the weight from the 'bandwidth-weights' line a
+ *   client would apply to N for position p if it had the guard
+ *   flag, Wpn the weight if it did not have the guard flag, and B the
+ *   measured bandwidth of N in the consensus.  Then instead of choosing
+ *   N for position p proportionally to Wpf*B or Wpn*B, clients should
+ *   choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
+ *
+ * This function fills the <b>guardfraction_bw</b> structure. It sets
+ * <b>guard_bw</b> to F*B and <b>non_guard_bw</b> to (1-F)*B.
+ */
+void
+guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
+                                  int orig_bandwidth,
+                                  uint32_t guardfraction_percentage)
+{
+  double guardfraction_fraction;
+
+  /* Turn the percentage into a fraction. */
+  tor_assert(guardfraction_percentage <= 100);
+  guardfraction_fraction = guardfraction_percentage / 100.0;
+
+  long guard_bw = tor_lround(guardfraction_fraction * orig_bandwidth);
+  tor_assert(guard_bw <= INT_MAX);
+
+  guardfraction_bw->guard_bw = (int) guard_bw;
+
+  guardfraction_bw->non_guard_bw = orig_bandwidth - (int) guard_bw;
+}
+
+/** Helper: Update the status of all entry guards, in whatever algorithm
+ * is used. Return true if we should stop using all previously generated
+ * circuits, by calling circuit_mark_all_unused_circs() and
+ * circuit_mark_all_dirty_circs_as_unusable().
+ */
+int
+guards_update_all(void)
+{
+  int mark_circuits = 0;
+  if (update_guard_selection_choice(get_options()))
+    mark_circuits = 1;
+
+  tor_assert(curr_guard_context);
+
+  if (entry_guards_update_all(curr_guard_context))
+    mark_circuits = 1;
+
+  return mark_circuits;
+}
+
+/** Helper: pick a guard for a circuit, with whatever algorithm is
+    used. */
+const node_t *
+guards_choose_guard(cpath_build_state_t *state,
+                    uint8_t purpose,
+                    circuit_guard_state_t **guard_state_out)
+{
+  const node_t *r = NULL;
+  const uint8_t *exit_id = NULL;
+  entry_guard_restriction_t *rst = NULL;
+
+  /* Only apply restrictions if we have a specific exit node in mind, and only
+   * if we are not doing vanguard circuits: we don't want to apply guard
+   * restrictions to vanguard circuits. */
+  if (state && !circuit_should_use_vanguards(purpose) &&
+      (exit_id = build_state_get_exit_rsa_id(state))) {
+    /* We're building to a targeted exit node, so that node can't be
+     * chosen as our guard for this circuit.  Remember that fact in a
+     * restriction. */
+    rst = guard_create_exit_restriction(exit_id);
+    tor_assert(rst);
+  }
+  if (entry_guard_pick_for_circuit(get_guard_selection_info(),
+                                   GUARD_USAGE_TRAFFIC,
+                                   rst,
+                                   &r,
+                                   guard_state_out) < 0) {
+    tor_assert(r == NULL);
+  }
+  return r;
+}
+
+/** Remove all currently listed entry guards for a given guard selection
+ * context.  This frees and replaces <b>gs</b>, so don't use <b>gs</b>
+ * after calling this function. */
+void
+remove_all_entry_guards_for_guard_selection(guard_selection_t *gs)
+{
+  // This function shouldn't exist. XXXX
+  tor_assert(gs != NULL);
+  char *old_name = tor_strdup(gs->name);
+  guard_selection_type_t old_type = gs->type;
+
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, entry, {
+    control_event_guard(entry->nickname, entry->identity, "DROPPED");
+  });
+
+  if (gs == curr_guard_context) {
+    curr_guard_context = NULL;
+  }
+
+  smartlist_remove(guard_contexts, gs);
+  guard_selection_free(gs);
+
+  gs = get_guard_selection_by_name(old_name, old_type, 1);
+  entry_guards_changed_for_guard_selection(gs);
+  tor_free(old_name);
+}
+
+/** Remove all currently listed entry guards, so new ones will be chosen.
+ *
+ * XXXX This function shouldn't exist -- it's meant to support the DROPGUARDS
+ * command, which is deprecated.
+ */
+void
+remove_all_entry_guards(void)
+{
+  remove_all_entry_guards_for_guard_selection(get_guard_selection_info());
+}
+
+/** Helper: pick a directory guard, with whatever algorithm is used. */
+const node_t *
+guards_choose_dirguard(uint8_t dir_purpose,
+                       circuit_guard_state_t **guard_state_out)
+{
+  const node_t *r = NULL;
+  entry_guard_restriction_t *rst = NULL;
+
+  /* If we are fetching microdescs, don't query outdated dirservers. */
+  if (dir_purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    rst = guard_create_dirserver_md_restriction();
+  }
+
+  if (entry_guard_pick_for_circuit(get_guard_selection_info(),
+                                   GUARD_USAGE_DIRGUARD,
+                                   rst,
+                                   &r,
+                                   guard_state_out) < 0) {
+    tor_assert(r == NULL);
+  }
+  return r;
+}
+
+/**
+ * If we're running with a constrained guard set, then maybe mark our guards
+ * usable.  Return 1 if we do; 0 if we don't.
+ */
+int
+guards_retry_optimistic(const or_options_t *options)
+{
+  if (! entry_list_is_constrained(options))
+    return 0;
+
+  mark_primary_guards_maybe_reachable(get_guard_selection_info());
+
+  return 1;
+}
+
+/**
+ * Check if we are missing any crucial dirinfo for the guard subsystem to
+ * work. Return NULL if everything went well, otherwise return a newly
+ * allocated string with an informative error message. In the latter case, use
+ * the genreal descriptor information <b>using_mds</b>, <b>num_present</b> and
+ * <b>num_usable</b> to improve the error message. */
+char *
+guard_selection_get_err_str_if_dir_info_missing(guard_selection_t *gs,
+                                        int using_mds,
+                                        int num_present, int num_usable)
+{
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  char *ret_str = NULL;
+  int n_missing_descriptors = 0;
+  int n_considered = 0;
+  int num_primary_to_check;
+
+  /* We want to check for the descriptor of at least the first two primary
+   * guards in our list, since these are the guards that we typically use for
+   * circuits. */
+  num_primary_to_check = get_n_primary_guards_to_use(GUARD_USAGE_TRAFFIC);
+  num_primary_to_check++;
+
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (guard->is_reachable == GUARD_REACHABLE_NO)
+      continue;
+    n_considered++;
+    if (!guard_has_descriptor(guard))
+      n_missing_descriptors++;
+    if (n_considered >= num_primary_to_check)
+      break;
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* If we are not missing any descriptors, return NULL. */
+  if (!n_missing_descriptors) {
+    return NULL;
+  }
+
+  /* otherwise return a helpful error string */
+  tor_asprintf(&ret_str, "We're missing descriptors for %d/%d of our "
+               "primary entry guards (total %sdescriptors: %d/%d).",
+               n_missing_descriptors, num_primary_to_check,
+               using_mds?"micro":"", num_present, num_usable);
+
+  return ret_str;
+}
+
+/** As guard_selection_have_enough_dir_info_to_build_circuits, but uses
+ * the default guard selection. */
+char *
+entry_guards_get_err_str_if_dir_info_missing(int using_mds,
+                                     int num_present, int num_usable)
+{
+  return guard_selection_get_err_str_if_dir_info_missing(
+                                                 get_guard_selection_info(),
+                                                 using_mds,
+                                                 num_present, num_usable);
+}
+
+/** Free one guard selection context */
+STATIC void
+guard_selection_free_(guard_selection_t *gs)
+{
+  if (!gs) return;
+
+  tor_free(gs->name);
+
+  if (gs->sampled_entry_guards) {
+    SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, e,
+                      entry_guard_free(e));
+    smartlist_free(gs->sampled_entry_guards);
+    gs->sampled_entry_guards = NULL;
+  }
+
+  smartlist_free(gs->confirmed_entry_guards);
+  smartlist_free(gs->primary_entry_guards);
+
+  tor_free(gs);
+}
+
+/** Release all storage held by the list of entry guards and related
+ * memory structs. */
+void
+entry_guards_free_all(void)
+{
+  /* Null out the default */
+  curr_guard_context = NULL;
+  /* Free all the guard contexts */
+  if (guard_contexts != NULL) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      guard_selection_free(gs);
+    } SMARTLIST_FOREACH_END(gs);
+    smartlist_free(guard_contexts);
+    guard_contexts = NULL;
+  }
+  circuit_build_times_free_timeouts(get_circuit_build_times_mutable());
+}
diff --git a/src/feature/client/entrynodes.h b/src/feature/client/entrynodes.h
new file mode 100644
index 0000000000..5f9b5bdcba
--- /dev/null
+++ b/src/feature/client/entrynodes.h
@@ -0,0 +1,639 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file entrynodes.h
+ * \brief Header file for circuitbuild.c.
+ **/
+
+#ifndef TOR_ENTRYNODES_H
+#define TOR_ENTRYNODES_H
+
+#include "lib/container/handles.h"
+
+/* Forward declare for guard_selection_t; entrynodes.c has the real struct */
+typedef struct guard_selection_s guard_selection_t;
+
+/* Forward declare for entry_guard_t; the real declaration is private. */
+typedef struct entry_guard_t entry_guard_t;
+
+/* Forward declaration for circuit_guard_state_t; the real declaration is
+   private. */
+typedef struct circuit_guard_state_t circuit_guard_state_t;
+
+/* Forward declaration for entry_guard_restriction_t; the real declaration is
+   private. */
+typedef struct entry_guard_restriction_t entry_guard_restriction_t;
+
+/* Information about a guard's pathbias status.
+ * These fields are used in circpathbias.c to try to detect entry
+ * nodes that are failing circuits at a suspicious frequency.
+ */
+typedef struct guard_pathbias_t {
+  unsigned int path_bias_noticed : 1; /**< Did we alert the user about path
+                                       * bias for this node already? */
+  unsigned int path_bias_warned : 1; /**< Did we alert the user about path bias
+                                      * for this node already? */
+  unsigned int path_bias_extreme : 1; /**< Did we alert the user about path
+                                       * bias for this node already? */
+  unsigned int path_bias_disabled : 1; /**< Have we disabled this node because
+                                        * of path bias issues? */
+  unsigned int path_bias_use_noticed : 1; /**< Did we alert the user about path
+                                       * use bias for this node already? */
+  unsigned int path_bias_use_extreme : 1; /**< Did we alert the user about path
+                                       * use bias for this node already? */
+
+  double circ_attempts; /**< Number of circuits this guard has "attempted" */
+  double circ_successes; /**< Number of successfully built circuits using
+                               * this guard as first hop. */
+  double successful_circuits_closed; /**< Number of circuits that carried
+                                        * streams successfully. */
+  double collapsed_circuits; /**< Number of fully built circuits that were
+                                 * remotely closed before any streams were
+                                 * attempted. */
+  double unusable_circuits; /**< Number of circuits for which streams were
+                                *  attempted, but none succeeded. */
+  double timeouts; /**< Number of 'right-censored' circuit timeouts for this
+                       * guard. */
+  double use_attempts; /**< Number of circuits we tried to use with streams */
+  double use_successes; /**< Number of successfully used circuits using
+                               * this guard as first hop. */
+} guard_pathbias_t;
+
+#if defined(ENTRYNODES_PRIVATE)
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/**
+ * @name values for entry_guard_t.is_reachable.
+ *
+ * See entry_guard_t.is_reachable for more information.
+ */
+/**@{*/
+#define GUARD_REACHABLE_NO    0
+#define GUARD_REACHABLE_YES   1
+#define GUARD_REACHABLE_MAYBE 2
+/**@}*/
+
+/** An entry_guard_t represents our information about a chosen long-term
+ * first hop, known as a "helper" node in the literature. We can't just
+ * use a node_t, since we want to remember these even when we
+ * don't have any directory info. */
+struct entry_guard_t {
+  HANDLE_ENTRY(entry_guard, entry_guard_t);
+
+  char nickname[MAX_HEX_NICKNAME_LEN+1];
+  char identity[DIGEST_LEN];
+  ed25519_public_key_t ed_id;
+
+  /**
+   * @name new guard selection algorithm fields.
+   *
+   * Only the new (prop271) algorithm uses these.  For a more full
+   * description of the algorithm, see the module documentation for
+   * entrynodes.c
+   */
+  /**@{*/
+
+  /* == Persistent fields, present for all sampled guards. */
+  /** When was this guard added to the sample? */
+  time_t sampled_on_date;
+  /** Since what date has this guard been "unlisted"?  A guard counts as
+   * unlisted if we have a live consensus that does not include it, or
+   * if we have a live consensus that does not include it as a usable
+   * guard.  This field is zero when the guard is listed. */
+  time_t unlisted_since_date; // can be zero
+  /** What version of Tor added this guard to the sample? */
+  char *sampled_by_version;
+  /** Is this guard listed right now? If this is set, then
+   * unlisted_since_date should be set too. */
+  unsigned currently_listed : 1;
+
+  /* == Persistent fields, for confirmed guards only */
+  /** When was this guard confirmed? (That is, when did we first use it
+   * successfully and decide to keep it?) This field is zero if this is not a
+   * confirmed guard. */
+  time_t confirmed_on_date; /* 0 if not confirmed */
+  /**
+   * In what order was this guard confirmed? Guards with lower indices
+   * appear earlier on the confirmed list.  If the confirmed list is compacted,
+   * this field corresponds to the index of this guard on the confirmed list.
+   *
+   * This field is set to -1 if this guard is not confirmed.
+   */
+  int confirmed_idx; /* -1 if not confirmed; otherwise the order that this
+                      * item should occur in the CONFIRMED_GUARDS ordered
+                      * list */
+
+  /**
+   * Which selection does this guard belong to?
+   */
+  char *selection_name;
+
+  /** Bridges only: address of the bridge. */
+  tor_addr_port_t *bridge_addr;
+
+  /* ==== Non-persistent fields. */
+  /* == These are used by sampled guards */
+  /** When did we last decide to try using this guard for a circuit? 0 for
+   * "not since we started up." */
+  time_t last_tried_to_connect;
+  /** How reachable do we consider this guard to be? One of
+   * GUARD_REACHABLE_NO, GUARD_REACHABLE_YES, or GUARD_REACHABLE_MAYBE. */
+  unsigned is_reachable : 2;
+  /** Boolean: true iff this guard is pending. A pending guard is one
+   * that we have an in-progress circuit through, and which we do not plan
+   * to try again until it either succeeds or fails. Primary guards can
+   * never be pending. */
+  unsigned is_pending : 1;
+  /** If true, don't write this guard to disk. (Used for bridges with unknown
+   * identities) */
+  unsigned is_persistent : 1;
+  /** When did we get the earliest connection failure for this guard?
+   * We clear this field on a successful connect.  We do _not_ clear it
+   * when we mark the guard as "MAYBE" reachable.
+   */
+  time_t failing_since;
+
+  /* == Set inclusion flags. */
+  /** If true, this guard is in the filtered set.  The filtered set includes
+   * all sampled guards that our configuration allows us to use. */
+  unsigned is_filtered_guard : 1;
+  /** If true, this guard is in the usable filtered set. The usable filtered
+   * set includes all filtered guards that are not believed to be
+   * unreachable. (That is, those for which is_reachable is not
+   * GUARD_REACHABLE_NO) */
+  unsigned is_usable_filtered_guard : 1;
+  unsigned is_primary:1;
+
+  /** This string holds any fields that we are maintaining because
+   * we saw them in the state, even if we don't understand them. */
+  char *extra_state_fields;
+
+  /** Backpointer to the guard selection that this guard belongs to.
+   * The entry_guard_t must never outlive its guard_selection. */
+  guard_selection_t *in_selection;
+  /**@}*/
+
+  /** Path bias information for this guard. */
+  guard_pathbias_t pb;
+};
+
+/**
+ * Possible rules for a guard selection to follow
+ */
+typedef enum guard_selection_type_t {
+  /** Infer the type of this selection from its name. */
+  GS_TYPE_INFER=0,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * complete list of guards in the consensus. */
+  GS_TYPE_NORMAL=1,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * configured bridges, and allowing it to grow as large as all the configured
+   * bridges */
+  GS_TYPE_BRIDGE,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * set of filtered nodes. */
+  GS_TYPE_RESTRICTED,
+} guard_selection_type_t;
+
+/**
+ * All of the the context for guard selection on a particular client.
+ *
+ * We maintain multiple guard selection contexts for a client, depending
+ * aspects on its current configuration -- whether an extremely
+ * restrictive EntryNodes is used, whether UseBridges is enabled, and so
+ * on.)
+ *
+ * See the module documentation for entrynodes.c for more information
+ * about guard selection algorithms.
+ */
+struct guard_selection_s {
+  /**
+   * The name for this guard-selection object. (Must not contain spaces).
+   */
+  char *name;
+
+  /**
+   * What rules does this guard-selection object follow?
+   */
+  guard_selection_type_t type;
+
+  /**
+   * A value of 1 means that primary_entry_guards is up-to-date with respect to
+   * the consensus and status info that we currently have; 0 means we need to
+   * recalculate it before using primary_entry_guards or the is_primary flag on
+   * any guard.
+   */
+  int primary_guards_up_to_date;
+
+  /**
+   * A list of the sampled entry guards, as entry_guard_t structures.
+   * Not in any particular order.  When we 'sample' a guard, we are
+   * noting it as a possible guard to pick in the future. The use of
+   * sampling here prevents us from being forced by an attacker to try
+   * every guard on the network. This list is persistent.
+   */
+  smartlist_t *sampled_entry_guards;
+
+  /**
+   * Ordered list (from highest to lowest priority) of guards that we
+   * have successfully contacted and decided to use. Every member of
+   * this list is a member of sampled_entry_guards. Every member should
+   * have confirmed_on_date set, and have confirmed_idx greater than
+   * any earlier member of the list.
+   *
+   * This list is persistent. It is a subset of the elements in
+   * sampled_entry_guards, and its pointers point to elements of
+   * sampled_entry_guards.
+   */
+  smartlist_t *confirmed_entry_guards;
+
+  /**
+   * Ordered list (from highest to lowest priority) of guards that we
+   * are willing to use the most happily.  These guards may or may not
+   * yet be confirmed yet.  If we can use one of these guards, we are
+   * probably not on a network that is trying to restrict our guard
+   * choices.
+   *
+   * This list is a subset of the elements in
+   * sampled_entry_guards, and its pointers point to elements of
+   * sampled_entry_guards.
+   */
+  smartlist_t *primary_entry_guards;
+
+  /** When did we last successfully build a circuit or use a circuit? */
+  time_t last_time_on_internet;
+
+  /** What confirmed_idx value should the next-added member of
+   * confirmed_entry_guards receive? */
+  int next_confirmed_idx;
+
+};
+
+struct entry_guard_handle_t;
+
+/** Types of restrictions we impose when picking guard nodes */
+typedef enum guard_restriction_type_t {
+  /* Don't pick the same guard node as our exit node (or its family) */
+  RST_EXIT_NODE = 0,
+  /* Don't pick dirguards that have previously shown to be outdated */
+  RST_OUTDATED_MD_DIRSERVER = 1
+} guard_restriction_type_t;
+
+/**
+ * A restriction to remember which entry guards are off-limits for a given
+ * circuit.
+ *
+ * Note: This mechanism is NOT for recording which guards are never to be
+ * used: only which guards cannot be used on <em>one particular circuit</em>.
+ */
+struct entry_guard_restriction_t {
+  /* What type of restriction are we imposing? */
+  guard_restriction_type_t type;
+
+  /* In case of restriction type RST_EXIT_NODE, the guard's RSA identity
+   * digest must not equal this; and it must not be in the same family as any
+   * node with this digest. */
+  uint8_t exclude_id[DIGEST_LEN];
+};
+
+/**
+ * Per-circuit state to track whether we'll be able to use the circuit.
+ */
+struct circuit_guard_state_t {
+  /** Handle to the entry guard object for this circuit. */
+  struct entry_guard_handle_t *guard;
+  /** The time at which <b>state</b> last changed. */
+  time_t state_set_at;
+  /** One of GUARD_CIRC_STATE_* */
+  uint8_t state;
+
+  /**
+   * A set of restrictions that were placed on this guard when we selected it
+   * for this particular circuit.  We need to remember the restrictions here,
+   * since any guard that breaks these restrictions will not block this
+   * circuit from becoming COMPLETE.
+   */
+  entry_guard_restriction_t *restrictions;
+};
+#endif /* defined(ENTRYNODES_PRIVATE) */
+
+/* Common entry points for old and new guard code */
+int guards_update_all(void);
+const node_t *guards_choose_guard(cpath_build_state_t *state,
+                                  uint8_t purpose,
+                                  circuit_guard_state_t **guard_state_out);
+const node_t *guards_choose_dirguard(uint8_t dir_purpose,
+                                     circuit_guard_state_t **guard_state_out);
+
+#if 1
+/* XXXX NM I would prefer that all of this stuff be private to
+ * entrynodes.c. */
+entry_guard_t *entry_guard_get_by_id_digest_for_guard_selection(
+    guard_selection_t *gs, const char *digest);
+entry_guard_t *entry_guard_get_by_id_digest(const char *digest);
+
+circuit_guard_state_t *
+get_guard_state_for_bridge_desc_fetch(const char *digest);
+
+void entry_guards_changed_for_guard_selection(guard_selection_t *gs);
+void entry_guards_changed(void);
+guard_selection_t * get_guard_selection_info(void);
+int num_live_entry_guards_for_guard_selection(
+    guard_selection_t *gs,
+    int for_directory);
+int num_live_entry_guards(int for_directory);
+#endif /* 1 */
+
+const node_t *entry_guard_find_node(const entry_guard_t *guard);
+const char *entry_guard_get_rsa_id_digest(const entry_guard_t *guard);
+const char *entry_guard_describe(const entry_guard_t *guard);
+guard_pathbias_t *entry_guard_get_pathbias_state(entry_guard_t *guard);
+
+/** Enum to specify how we're going to use a given guard, when we're picking
+ * one for immediate use. */
+typedef enum {
+  GUARD_USAGE_TRAFFIC = 0,
+  GUARD_USAGE_DIRGUARD = 1
+} guard_usage_t;
+
+#define circuit_guard_state_free(val) \
+  FREE_AND_NULL(circuit_guard_state_t, circuit_guard_state_free_, (val))
+
+void circuit_guard_state_free_(circuit_guard_state_t *state);
+int entry_guard_pick_for_circuit(guard_selection_t *gs,
+                                 guard_usage_t usage,
+                                 entry_guard_restriction_t *rst,
+                                 const node_t **chosen_node_out,
+                                 circuit_guard_state_t **guard_state_out);
+
+/* We just connected to an entry guard. What should we do with the circuit? */
+typedef enum {
+  GUARD_USABLE_NEVER = -1, /* Never use the circuit */
+  GUARD_MAYBE_USABLE_LATER = 0, /* Keep it. We might use it in the future */
+  GUARD_USABLE_NOW = 1, /* Use it right now */
+} guard_usable_t;
+
+guard_usable_t entry_guard_succeeded(circuit_guard_state_t **guard_state_p);
+void entry_guard_failed(circuit_guard_state_t **guard_state_p);
+void entry_guard_cancel(circuit_guard_state_t **guard_state_p);
+void entry_guard_chan_failed(channel_t *chan);
+int entry_guards_update_all(guard_selection_t *gs);
+int entry_guards_upgrade_waiting_circuits(guard_selection_t *gs,
+                                          const smartlist_t *all_circuits,
+                                          smartlist_t *newly_complete_out);
+int entry_guard_state_should_expire(circuit_guard_state_t *guard_state);
+void entry_guards_note_internet_connectivity(guard_selection_t *gs);
+
+int update_guard_selection_choice(const or_options_t *options);
+
+int entry_guard_could_succeed(const circuit_guard_state_t *guard_state);
+
+MOCK_DECL(int,num_bridges_usable,(int use_maybe_reachable));
+
+#ifdef ENTRYNODES_PRIVATE
+/**
+ * @name Default values for the parameters for the new (prop271) entry guard
+ * algorithm.
+ */
+/**@{*/
+/**
+ * We never let our sampled guard set grow larger than this percentage
+ * of the guards on the network.
+ */
+#define DFLT_MAX_SAMPLE_THRESHOLD_PERCENT 20
+/**
+ * We never let our sampled guard set grow larger than this number of
+ * guards.
+ */
+#define DFLT_MAX_SAMPLE_SIZE 60
+/**
+ * We always try to make our sample contain at least this many guards.
+ */
+#define DFLT_MIN_FILTERED_SAMPLE_SIZE 20
+/**
+ * If a guard is unlisted for this many days in a row, we remove it.
+ */
+#define DFLT_REMOVE_UNLISTED_GUARDS_AFTER_DAYS 20
+/**
+ * We remove unconfirmed guards from the sample after this many days,
+ * regardless of whether they are listed or unlisted.
+ */
+#define DFLT_GUARD_LIFETIME_DAYS 120
+/**
+ * We remove confirmed guards from the sample if they were sampled
+ * GUARD_LIFETIME_DAYS ago and confirmed this many days ago.
+ */
+#define DFLT_GUARD_CONFIRMED_MIN_LIFETIME_DAYS 60
+/**
+ * How many guards do we try to keep on our primary guard list?
+ */
+#define DFLT_N_PRIMARY_GUARDS 3
+/**
+ * Of the live guards on the primary guard list, how many do we consider when
+ * choosing a guard to use?
+ */
+#define DFLT_N_PRIMARY_GUARDS_TO_USE 1
+/**
+ * As DFLT_N_PRIMARY_GUARDS, but for choosing which directory guard to use.
+ */
+#define DFLT_N_PRIMARY_DIR_GUARDS_TO_USE 3
+/**
+ * If we haven't successfully built or used a circuit in this long, then
+ * consider that the internet is probably down.
+ */
+#define DFLT_INTERNET_LIKELY_DOWN_INTERVAL (10*60)
+/**
+ * If we're trying to connect to a nonprimary guard for at least this
+ * many seconds, and we haven't gotten the connection to work, we will treat
+ * lower-priority guards as usable.
+ */
+#define DFLT_NONPRIMARY_GUARD_CONNECT_TIMEOUT 15
+/**
+ * If a circuit has been sitting around in 'waiting for better guard' state
+ * for at least this long, we'll expire it.
+ */
+#define DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT (10*60)
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in a restricted setting.
+ */
+#define DFLT_MEANINGFUL_RESTRICTION_PERCENT 20
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in an extremely restricted setting, and should warn.
+ */
+#define DFLT_EXTREME_RESTRICTION_PERCENT 1
+/**@}*/
+
+STATIC double get_max_sample_threshold(void);
+STATIC int get_max_sample_size_absolute(void);
+STATIC int get_min_filtered_sample_size(void);
+STATIC int get_remove_unlisted_guards_after_days(void);
+STATIC int get_guard_lifetime(void);
+STATIC int get_guard_confirmed_min_lifetime(void);
+STATIC int get_n_primary_guards(void);
+STATIC int get_n_primary_guards_to_use(guard_usage_t usage);
+STATIC int get_internet_likely_down_interval(void);
+STATIC int get_nonprimary_guard_connect_timeout(void);
+STATIC int get_nonprimary_guard_idle_timeout(void);
+STATIC double get_meaningful_restriction_threshold(void);
+STATIC double get_extreme_restriction_threshold(void);
+
+HANDLE_DECL(entry_guard, entry_guard_t, STATIC)
+#define entry_guard_handle_free(h)    \
+  FREE_AND_NULL(entry_guard_handle_t, entry_guard_handle_free_, (h))
+
+STATIC guard_selection_type_t guard_selection_infer_type(
+                           guard_selection_type_t type_in,
+                           const char *name);
+STATIC guard_selection_t *guard_selection_new(const char *name,
+                                              guard_selection_type_t type);
+STATIC guard_selection_t *get_guard_selection_by_name(
+          const char *name, guard_selection_type_t type, int create_if_absent);
+STATIC void guard_selection_free_(guard_selection_t *gs);
+#define guard_selection_free(gs) \
+  FREE_AND_NULL(guard_selection_t, guard_selection_free_, (gs))
+MOCK_DECL(STATIC int, entry_guard_is_listed,
+          (guard_selection_t *gs, const entry_guard_t *guard));
+STATIC const char *choose_guard_selection(const or_options_t *options,
+                                        const networkstatus_t *ns,
+                                        const guard_selection_t *old_selection,
+                                        guard_selection_type_t *type_out);
+STATIC entry_guard_t *get_sampled_guard_with_id(guard_selection_t *gs,
+                                                const uint8_t *rsa_id);
+
+MOCK_DECL(STATIC time_t, randomize_time, (time_t now, time_t max_backdate));
+
+MOCK_DECL(STATIC circuit_guard_state_t *,
+          circuit_guard_state_new,(entry_guard_t *guard, unsigned state,
+                                   entry_guard_restriction_t *rst));
+
+STATIC entry_guard_t *entry_guard_add_to_sample(guard_selection_t *gs,
+                                                const node_t *node);
+STATIC entry_guard_t *entry_guards_expand_sample(guard_selection_t *gs);
+STATIC char *entry_guard_encode_for_state(entry_guard_t *guard);
+STATIC entry_guard_t *entry_guard_parse_from_state(const char *s);
+#define entry_guard_free(e) \
+  FREE_AND_NULL(entry_guard_t, entry_guard_free_, (e))
+STATIC void entry_guard_free_(entry_guard_t *e);
+STATIC void entry_guards_update_filtered_sets(guard_selection_t *gs);
+STATIC int entry_guards_all_primary_guards_are_down(guard_selection_t *gs);
+/**
+ * @name Flags for sample_reachable_filtered_entry_guards()
+ */
+/**@{*/
+#define SAMPLE_EXCLUDE_CONFIRMED   (1u<<0)
+#define SAMPLE_EXCLUDE_PRIMARY     (1u<<1)
+#define SAMPLE_EXCLUDE_PENDING     (1u<<2)
+#define SAMPLE_NO_UPDATE_PRIMARY   (1u<<3)
+#define SAMPLE_EXCLUDE_NO_DESCRIPTOR (1u<<4)
+/**@}*/
+STATIC entry_guard_t *sample_reachable_filtered_entry_guards(
+                                    guard_selection_t *gs,
+                                    const entry_guard_restriction_t *rst,
+                                    unsigned flags);
+STATIC void entry_guard_consider_retry(entry_guard_t *guard);
+STATIC void make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard);
+STATIC void entry_guards_update_confirmed(guard_selection_t *gs);
+STATIC void entry_guards_update_primary(guard_selection_t *gs);
+STATIC int num_reachable_filtered_guards(const guard_selection_t *gs,
+                                         const entry_guard_restriction_t *rst);
+STATIC void sampled_guards_update_from_consensus(guard_selection_t *gs);
+/**
+ * @name Possible guard-states for a circuit.
+ */
+/**@{*/
+/** State for a circuit that can (so far as the guard subsystem is
+ * concerned) be used for actual traffic as soon as it is successfully
+ * opened. */
+#define GUARD_CIRC_STATE_USABLE_ON_COMPLETION 1
+/** State for an non-open circuit that we shouldn't use for actual
+ * traffic, when it completes, unless other circuits to preferable
+ * guards fail. */
+#define GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD 2
+/** State for an open circuit that we shouldn't use for actual traffic
+ * unless other circuits to preferable guards fail. */
+#define GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD 3
+/** State for a circuit that can (so far as the guard subsystem is
+ * concerned) be used for actual traffic. */
+#define GUARD_CIRC_STATE_COMPLETE 4
+/** State for a circuit that is unusable, and will not become usable. */
+#define GUARD_CIRC_STATE_DEAD 5
+/**@}*/
+STATIC void entry_guards_note_guard_failure(guard_selection_t *gs,
+                                            entry_guard_t *guard);
+STATIC entry_guard_t *select_entry_guard_for_circuit(guard_selection_t *gs,
+                                          guard_usage_t usage,
+                                          const entry_guard_restriction_t *rst,
+                                          unsigned *state_out);
+STATIC void mark_primary_guards_maybe_reachable(guard_selection_t *gs);
+STATIC unsigned entry_guards_note_guard_success(guard_selection_t *gs,
+                                                entry_guard_t *guard,
+                                                unsigned old_state);
+STATIC int entry_guard_has_higher_priority(entry_guard_t *a, entry_guard_t *b);
+STATIC char *getinfo_helper_format_single_entry_guard(const entry_guard_t *e);
+
+STATIC entry_guard_restriction_t *guard_create_exit_restriction(
+                                                      const uint8_t *exit_id);
+
+STATIC entry_guard_restriction_t *guard_create_dirserver_md_restriction(void);
+
+STATIC void entry_guard_restriction_free_(entry_guard_restriction_t *rst);
+#define entry_guard_restriction_free(rst)  \
+  FREE_AND_NULL(entry_guard_restriction_t, \
+                entry_guard_restriction_free_, (rst))
+
+#endif /* defined(ENTRYNODES_PRIVATE) */
+
+void remove_all_entry_guards_for_guard_selection(guard_selection_t *gs);
+void remove_all_entry_guards(void);
+
+struct bridge_info_t;
+void entry_guard_learned_bridge_identity(const tor_addr_port_t *addrport,
+                                         const uint8_t *rsa_id_digest);
+
+int entry_list_is_constrained(const or_options_t *options);
+int guards_retry_optimistic(const or_options_t *options);
+int entry_guards_parse_state_for_guard_selection(
+    guard_selection_t *gs, or_state_t *state, int set, char **msg);
+int entry_guards_parse_state(or_state_t *state, int set, char **msg);
+void entry_guards_update_state(or_state_t *state);
+int getinfo_helper_entry_guards(control_connection_t *conn,
+                                const char *question, char **answer,
+                                const char **errmsg);
+
+int entries_known_but_down(const or_options_t *options);
+void entries_retry_all(const or_options_t *options);
+
+char *entry_guards_get_err_str_if_dir_info_missing(int using_mds,
+                                           int num_present, int num_usable);
+char *guard_selection_get_err_str_if_dir_info_missing(guard_selection_t *gs,
+                                              int using_mds,
+                                              int num_present, int num_usable);
+
+void entry_guards_free_all(void);
+
+double pathbias_get_close_success_count(entry_guard_t *guard);
+double pathbias_get_use_success_count(entry_guard_t *guard);
+
+/** Contains the bandwidth of a relay as a guard and as a non-guard
+ *  after the guardfraction has been considered. */
+typedef struct guardfraction_bandwidth_t {
+  /** Bandwidth as a guard after guardfraction has been considered. */
+  int guard_bw;
+  /** Bandwidth as a non-guard after guardfraction has been considered. */
+  int non_guard_bw;
+} guardfraction_bandwidth_t;
+
+int should_apply_guardfraction(const networkstatus_t *ns);
+
+void
+guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
+                                  int orig_bandwidth,
+                                  uint32_t guardfraction_percentage);
+
+#endif /* !defined(TOR_ENTRYNODES_H) */
diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c
new file mode 100644
index 0000000000..1d3cb7b951
--- /dev/null
+++ b/src/feature/client/transports.c
@@ -0,0 +1,1738 @@
+/* Copyright (c) 2011-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file transports.c
+ * \brief Pluggable Transports related code.
+ *
+ * \details
+ * Each managed proxy is represented by a <b>managed_proxy_t</b>.
+ * Each managed proxy can support multiple transports.
+ * Each managed proxy gets configured through a multistep process.
+ *
+ * ::managed_proxy_list contains all the managed proxies this tor
+ * instance is supporting.
+ * In the ::managed_proxy_list there are ::unconfigured_proxies_n
+ * managed proxies that are still unconfigured.
+ *
+ * In every run_scheduled_event() tick, we attempt to launch and then
+ * configure the unconfiged managed proxies, using the configuration
+ * protocol defined in the 180_pluggable_transport.txt proposal. A
+ * managed proxy might need several ticks to get fully configured.
+ *
+ * When a managed proxy is fully configured, we register all its
+ * transports to the circuitbuild.c subsystem. At that point the
+ * transports are owned by the circuitbuild.c subsystem.
+ *
+ * When a managed proxy fails to follow the 180 configuration
+ * protocol, it gets marked as broken and gets destroyed.
+ *
+ * <b>In a little more detail:</b>
+ *
+ * While we are serially parsing torrc, we store all the transports
+ * that a proxy should spawn in its <em>transports_to_launch</em>
+ * element.
+ *
+ * When we finish reading the torrc, we spawn the managed proxy and
+ * expect {S,C}METHOD lines from its output. We add transports
+ * described by METHOD lines to its <em>transports</em> element, as
+ * transport_t structs.
+ *
+ * When the managed proxy stops spitting METHOD lines (signified by a
+ * '{S,C}METHODS DONE' message) we pass copies of its transports to
+ * the bridge subsystem. We keep copies of the 'transport_t's on the
+ * managed proxy to be able to associate the proxy with its
+ * transports, and we pass copies to the bridge subsystem so that
+ * transports can be associated with bridges.
+ * [ XXX We should try see whether the two copies are really needed
+ * and maybe cut it into a single copy of the 'transport_t' shared
+ * between the managed proxy and the bridge subsystem. Preliminary
+ * analysis shows that both copies are needed with the current code
+ * logic, because of race conditions that can cause dangling
+ * pointers. ]
+ *
+ * <b>In even more detail, this is what happens when a config read
+ * (like a SIGHUP or a SETCONF) occurs:</b>
+ *
+ * We immediately destroy all unconfigured proxies (We shouldn't have
+ * unconfigured proxies in the first place, except when the config
+ * read happens immediately after tor is launched.).
+ *
+ * We mark all managed proxies and transports to signify that they
+ * must be removed if they don't contribute by the new torrc
+ * (we mark using the <b>marked_for_removal</b> element).
+ * We also mark all managed proxies to signify that they might need to
+ * be restarted so that they end up supporting all the transports the
+ * new torrc wants them to support
+ * (we mark using the <b>was_around_before_config_read</b> element).
+ * We also clear their <b>transports_to_launch</b> list so that we can
+ * put there the transports we need to launch according to the new
+ * torrc.
+ *
+ * We then start parsing torrc again.
+ *
+ * Everytime we encounter a transport line using a managed proxy that
+ * was around before the config read, we cleanse that proxy from the
+ * removal mark.  We also toggle the <b>check_if_restarts_needed</b>
+ * flag, so that on the next <b>pt_configure_remaining_proxies</b>
+ * tick, we investigate whether we need to restart the proxy so that
+ * it also spawns the new transports.  If the post-config-read
+ * <b>transports_to_launch</b> list is identical to the pre-config-read
+ * one, it means that no changes were introduced to this proxy during
+ * the config read and no restart has to take place.
+ *
+ * During the post-config-read torrc parsing, we unmark all transports
+ * spawned by managed proxies that we find in our torrc.
+ * We do that so that if we don't need to restart a managed proxy, we
+ * can continue using its old transports normally.
+ * If we end up restarting the proxy, we destroy and unregister all
+ * old transports from the circuitbuild.c subsystem.
+ **/
+
+#define PT_PRIVATE
+#include "or/or.h"
+#include "or/bridges.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/circuitbuild.h"
+#include "or/transports.h"
+#include "or/router.h"
+#include "or/statefile.h"
+#include "or/connection_or.h"
+#include "or/ext_orport.h"
+#include "or/control.h"
+
+#include "lib/process/env.h"
+#include "lib/process/subprocess.h"
+
+static process_environment_t *
+create_managed_proxy_environment(const managed_proxy_t *mp);
+
+static inline int proxy_configuration_finished(const managed_proxy_t *mp);
+
+static void handle_finished_proxy(managed_proxy_t *mp);
+static void parse_method_error(const char *line, int is_server_method);
+#define parse_server_method_error(l) parse_method_error(l, 1)
+#define parse_client_method_error(l) parse_method_error(l, 0)
+
+/** Managed proxy protocol strings */
+#define PROTO_ENV_ERROR "ENV-ERROR"
+#define PROTO_NEG_SUCCESS "VERSION"
+#define PROTO_NEG_FAIL "VERSION-ERROR no-version"
+#define PROTO_CMETHOD "CMETHOD"
+#define PROTO_SMETHOD "SMETHOD"
+#define PROTO_CMETHOD_ERROR "CMETHOD-ERROR"
+#define PROTO_SMETHOD_ERROR "SMETHOD-ERROR"
+#define PROTO_CMETHODS_DONE "CMETHODS DONE"
+#define PROTO_SMETHODS_DONE "SMETHODS DONE"
+#define PROTO_PROXY_DONE "PROXY DONE"
+#define PROTO_PROXY_ERROR "PROXY-ERROR"
+
+/** The first and only supported - at the moment - configuration
+    protocol version. */
+#define PROTO_VERSION_ONE 1
+
+/** A list of pluggable transports found in torrc. */
+static smartlist_t *transport_list = NULL;
+
+/** Returns a transport_t struct for a transport proxy supporting the
+    protocol <b>name</b> listening at <b>addr</b>:<b>port</b> using
+    SOCKS version <b>socks_ver</b>. */
+STATIC transport_t *
+transport_new(const tor_addr_t *addr, uint16_t port,
+              const char *name, int socks_ver,
+              const char *extra_info_args)
+{
+  transport_t *t = tor_malloc_zero(sizeof(transport_t));
+
+  tor_addr_copy(&t->addr, addr);
+  t->port = port;
+  t->name = tor_strdup(name);
+  t->socks_version = socks_ver;
+  if (extra_info_args)
+    t->extra_info_args = tor_strdup(extra_info_args);
+
+  return t;
+}
+
+/** Free the pluggable transport struct <b>transport</b>. */
+void
+transport_free_(transport_t *transport)
+{
+  if (!transport)
+    return;
+
+  tor_free(transport->name);
+  tor_free(transport->extra_info_args);
+  tor_free(transport);
+}
+
+/** Mark every entry of the transport list to be removed on our next call to
+ * sweep_transport_list unless it has first been un-marked. */
+void
+mark_transport_list(void)
+{
+  if (!transport_list)
+    transport_list = smartlist_new();
+  SMARTLIST_FOREACH(transport_list, transport_t *, t,
+                    t->marked_for_removal = 1);
+}
+
+/** Remove every entry of the transport list that was marked with
+ * mark_transport_list if it has not subsequently been un-marked. */
+void
+sweep_transport_list(void)
+{
+  if (!transport_list)
+    transport_list = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(transport_list, transport_t *, t) {
+    if (t->marked_for_removal) {
+      SMARTLIST_DEL_CURRENT(transport_list, t);
+      transport_free(t);
+    }
+  } SMARTLIST_FOREACH_END(t);
+}
+
+/** Initialize the pluggable transports list to empty, creating it if
+ *  needed. */
+static void
+clear_transport_list(void)
+{
+  if (!transport_list)
+    transport_list = smartlist_new();
+  SMARTLIST_FOREACH(transport_list, transport_t *, t, transport_free(t));
+  smartlist_clear(transport_list);
+}
+
+/** Return a deep copy of <b>transport</b>. */
+static transport_t *
+transport_copy(const transport_t *transport)
+{
+  transport_t *new_transport = NULL;
+
+  tor_assert(transport);
+
+  new_transport = tor_malloc_zero(sizeof(transport_t));
+
+  new_transport->socks_version = transport->socks_version;
+  new_transport->name = tor_strdup(transport->name);
+  tor_addr_copy(&new_transport->addr, &transport->addr);
+  new_transport->port = transport->port;
+  new_transport->marked_for_removal = transport->marked_for_removal;
+
+  return new_transport;
+}
+
+/** Returns the transport in our transport list that has the name <b>name</b>.
+ *  Else returns NULL. */
+MOCK_IMPL(transport_t *,
+transport_get_by_name,(const char *name))
+{
+  tor_assert(name);
+
+  if (!transport_list)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(transport_list, transport_t *, transport) {
+    if (!strcmp(transport->name, name))
+      return transport;
+  } SMARTLIST_FOREACH_END(transport);
+
+  return NULL;
+}
+
+/** Resolve any conflicts that the insertion of transport <b>t</b>
+ *  might cause.
+ *  Return 0 if <b>t</b> is OK and should be registered, 1 if there is
+ *  a transport identical to <b>t</b> already registered and -1 if
+ *  <b>t</b> cannot be added due to conflicts. */
+static int
+transport_resolve_conflicts(const transport_t *t)
+{
+  /* This is how we resolve transport conflicts:
+
+     If there is already a transport with the same name and addrport,
+     we either have duplicate torrc lines OR we are here post-HUP and
+     this transport was here pre-HUP as well. In any case, mark the
+     old transport so that it doesn't get removed and ignore the new
+     one. Our caller has to free the new transport so we return '1' to
+     signify this.
+
+     If there is already a transport with the same name but different
+     addrport:
+     * if it's marked for removal, it means that it either has a lower
+     priority than 't' in torrc (otherwise the mark would have been
+     cleared by the paragraph above), or it doesn't exist at all in
+     the post-HUP torrc. We destroy the old transport and register 't'.
+     * if it's *not* marked for removal, it means that it was newly
+     added in the post-HUP torrc or that it's of higher priority, in
+     this case we ignore 't'. */
+  transport_t *t_tmp = transport_get_by_name(t->name);
+  if (t_tmp) { /* same name */
+    if (tor_addr_eq(&t->addr, &t_tmp->addr) && (t->port == t_tmp->port)) {
+      /* same name *and* addrport */
+      t_tmp->marked_for_removal = 0;
+      return 1;
+    } else { /* same name but different addrport */
+      char *new_transport_addrport =
+        tor_strdup(fmt_addrport(&t->addr, t->port));
+      if (t_tmp->marked_for_removal) { /* marked for removal */
+        log_notice(LD_GENERAL, "You tried to add transport '%s' at '%s' "
+                   "but there was already a transport marked for deletion at "
+                   "'%s'. We deleted the old transport and registered the "
+                   "new one.", t->name, new_transport_addrport,
+                   fmt_addrport(&t_tmp->addr, t_tmp->port));
+        smartlist_remove(transport_list, t_tmp);
+        transport_free(t_tmp);
+        tor_free(new_transport_addrport);
+      } else { /* *not* marked for removal */
+        log_notice(LD_GENERAL, "You tried to add transport '%s' at '%s' "
+                   "but the same transport already exists at '%s'. "
+                   "Skipping.", t->name, new_transport_addrport,
+                   fmt_addrport(&t_tmp->addr, t_tmp->port));
+        tor_free(new_transport_addrport);
+        return -1;
+      }
+      tor_free(new_transport_addrport);
+    }
+  }
+
+  return 0;
+}
+
+/** Add transport <b>t</b> to the internal list of pluggable
+ *  transports.
+ *  Returns 0 if the transport was added correctly, 1 if the same
+ *  transport was already registered (in this case the caller must
+ *  free the transport) and -1 if there was an error.  */
+static int
+transport_add(transport_t *t)
+{
+  int r;
+  tor_assert(t);
+
+  r = transport_resolve_conflicts(t);
+
+  switch (r) {
+  case 0: /* should register transport */
+    if (!transport_list)
+      transport_list = smartlist_new();
+    smartlist_add(transport_list, t);
+    return 0;
+  default: /* let our caller know the return code */
+    return r;
+  }
+}
+
+/** Remember a new pluggable transport proxy at <b>addr</b>:<b>port</b>.
+ *  <b>name</b> is set to the name of the protocol this proxy uses.
+ *  <b>socks_ver</b> is set to the SOCKS version of the proxy. */
+MOCK_IMPL(int,
+transport_add_from_config, (const tor_addr_t *addr, uint16_t port,
+                            const char *name, int socks_ver))
+{
+  transport_t *t = transport_new(addr, port, name, socks_ver, NULL);
+
+  int r = transport_add(t);
+
+  switch (r) {
+  case -1:
+  default:
+    log_notice(LD_GENERAL, "Could not add transport %s at %s. Skipping.",
+               t->name, fmt_addrport(&t->addr, t->port));
+    transport_free(t);
+    return -1;
+  case 1:
+    log_info(LD_GENERAL, "Successfully registered transport %s at %s.",
+             t->name, fmt_addrport(&t->addr, t->port));
+     transport_free(t); /* falling */
+     return 0;
+  case 0:
+    log_info(LD_GENERAL, "Successfully registered transport %s at %s.",
+             t->name, fmt_addrport(&t->addr, t->port));
+    return 0;
+  }
+}
+
+/** List of unconfigured managed proxies. */
+static smartlist_t *managed_proxy_list = NULL;
+/** Number of still unconfigured proxies. */
+static int unconfigured_proxies_n = 0;
+/** Boolean: True iff we might need to restart some proxies. */
+static int check_if_restarts_needed = 0;
+
+/** Return true if there are still unconfigured managed proxies, or proxies
+ * that need restarting. */
+int
+pt_proxies_configuration_pending(void)
+{
+  return unconfigured_proxies_n || check_if_restarts_needed;
+}
+
+/** Assert that the unconfigured_proxies_n value correctly matches the number
+ * of proxies in a state other than PT_PROTO_COMPLETE. */
+static void
+assert_unconfigured_count_ok(void)
+{
+  int n_completed = 0;
+  if (!managed_proxy_list) {
+    tor_assert(unconfigured_proxies_n == 0);
+    return;
+  }
+
+  SMARTLIST_FOREACH(managed_proxy_list, managed_proxy_t *, mp, {
+      if (mp->conf_state == PT_PROTO_COMPLETED)
+        ++n_completed;
+  });
+
+  tor_assert(n_completed + unconfigured_proxies_n ==
+             smartlist_len(managed_proxy_list));
+}
+
+/** Return true if <b>mp</b> has the same argv as <b>proxy_argv</b> */
+static int
+managed_proxy_has_argv(const managed_proxy_t *mp, char **proxy_argv)
+{
+  char **tmp1=proxy_argv;
+  char **tmp2=mp->argv;
+
+  tor_assert(tmp1);
+  tor_assert(tmp2);
+
+  while (*tmp1 && *tmp2) {
+    if (strcmp(*tmp1++, *tmp2++))
+      return 0;
+  }
+
+  if (!*tmp1 && !*tmp2)
+    return 1;
+
+  return 0;
+}
+
+/** Return a managed proxy with the same argv as <b>proxy_argv</b>.
+ *  If no such managed proxy exists, return NULL. */
+static managed_proxy_t *
+get_managed_proxy_by_argv_and_type(char **proxy_argv, int is_server)
+{
+  if (!managed_proxy_list)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(managed_proxy_list,  managed_proxy_t *, mp) {
+    if (managed_proxy_has_argv(mp, proxy_argv) &&
+        mp->is_server == is_server)
+      return mp;
+  } SMARTLIST_FOREACH_END(mp);
+
+  return NULL;
+}
+
+/** Add <b>transport</b> to managed proxy <b>mp</b>. */
+static void
+add_transport_to_proxy(const char *transport, managed_proxy_t *mp)
+{
+  tor_assert(mp->transports_to_launch);
+  if (!smartlist_contains_string(mp->transports_to_launch, transport))
+    smartlist_add_strdup(mp->transports_to_launch, transport);
+}
+
+/** Called when a SIGHUP occurs. Returns true if managed proxy
+ *  <b>mp</b> needs to be restarted after the SIGHUP, based on the new
+ *  torrc. */
+static int
+proxy_needs_restart(const managed_proxy_t *mp)
+{
+  int ret = 1;
+  char* proxy_uri;
+
+  /* If the PT proxy config has changed, then all existing pluggable transports
+   * should be restarted.
+   */
+
+  proxy_uri = get_pt_proxy_uri();
+  if (strcmp_opt(proxy_uri, mp->proxy_uri) != 0)
+    goto needs_restart;
+
+  /* mp->transport_to_launch is populated with the names of the
+     transports that must be launched *after* the SIGHUP.
+     mp->transports is populated with the transports that were
+     launched *before* the SIGHUP.
+
+     Check if all the transports that need to be launched are already
+     launched: */
+
+  tor_assert(smartlist_len(mp->transports_to_launch) > 0);
+  tor_assert(mp->conf_state == PT_PROTO_COMPLETED);
+
+  if (smartlist_len(mp->transports_to_launch) != smartlist_len(mp->transports))
+    goto needs_restart;
+
+  SMARTLIST_FOREACH_BEGIN(mp->transports, const transport_t *, t) {
+    if (!smartlist_contains_string(mp->transports_to_launch, t->name))
+      goto needs_restart;
+
+  } SMARTLIST_FOREACH_END(t);
+
+  ret = 0;
+ needs_restart:
+  tor_free(proxy_uri);
+  return ret;
+}
+
+/** Managed proxy <b>mp</b> must be restarted. Do all the necessary
+ *  preparations and then flag its state so that it will be relaunched
+ *  in the next tick. */
+static void
+proxy_prepare_for_restart(managed_proxy_t *mp)
+{
+  transport_t *t_tmp = NULL;
+
+  tor_assert(mp->conf_state == PT_PROTO_COMPLETED);
+
+  /* destroy the process handle and terminate the process. */
+  tor_process_handle_destroy(mp->process_handle, 1);
+  mp->process_handle = NULL;
+
+  /* destroy all its registered transports, since we will no longer
+     use them. */
+  SMARTLIST_FOREACH_BEGIN(mp->transports, const transport_t *, t) {
+    t_tmp = transport_get_by_name(t->name);
+    if (t_tmp)
+      t_tmp->marked_for_removal = 1;
+  } SMARTLIST_FOREACH_END(t);
+  sweep_transport_list();
+
+  /* free the transport in mp->transports */
+  SMARTLIST_FOREACH(mp->transports, transport_t *, t, transport_free(t));
+  smartlist_clear(mp->transports);
+
+  /* Reset the proxy's HTTPS/SOCKS proxy */
+  tor_free(mp->proxy_uri);
+  mp->proxy_uri = get_pt_proxy_uri();
+  mp->proxy_supported = 0;
+
+  /* flag it as an infant proxy so that it gets launched on next tick */
+  mp->conf_state = PT_PROTO_INFANT;
+  unconfigured_proxies_n++;
+}
+
+/** Launch managed proxy <b>mp</b>. */
+static int
+launch_managed_proxy(managed_proxy_t *mp)
+{
+  int retval;
+
+  process_environment_t *env = create_managed_proxy_environment(mp);
+
+#ifdef _WIN32
+  /* Passing NULL as lpApplicationName makes Windows search for the .exe */
+  retval = tor_spawn_background(NULL,
+                                (const char **)mp->argv,
+                                env,
+                                &mp->process_handle);
+#else /* !(defined(_WIN32)) */
+  retval = tor_spawn_background(mp->argv[0],
+                                (const char **)mp->argv,
+                                env,
+                                &mp->process_handle);
+#endif /* defined(_WIN32) */
+
+  process_environment_free(env);
+
+  if (retval == PROCESS_STATUS_ERROR) {
+    log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.",
+             mp->argv[0]);
+    return -1;
+  }
+
+  log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.",
+           mp->argv[0], tor_process_get_pid(mp->process_handle));
+
+  mp->conf_state = PT_PROTO_LAUNCHED;
+
+  return 0;
+}
+
+/** Check if any of the managed proxies we are currently trying to
+ *  configure has anything new to say. */
+void
+pt_configure_remaining_proxies(void)
+{
+  int at_least_a_proxy_config_finished = 0;
+  smartlist_t *tmp = smartlist_new();
+
+  log_debug(LD_CONFIG, "Configuring remaining managed proxies (%d)!",
+            unconfigured_proxies_n);
+
+  /* Iterate over tmp, not managed_proxy_list, since configure_proxy can
+   * remove elements from managed_proxy_list. */
+  smartlist_add_all(tmp, managed_proxy_list);
+
+  assert_unconfigured_count_ok();
+
+  SMARTLIST_FOREACH_BEGIN(tmp,  managed_proxy_t *, mp) {
+    tor_assert(mp->conf_state != PT_PROTO_BROKEN &&
+               mp->conf_state != PT_PROTO_FAILED_LAUNCH);
+
+    if (mp->was_around_before_config_read) {
+      /* This proxy is marked by a config read. Check whether we need
+         to restart it. */
+
+      mp->was_around_before_config_read = 0;
+
+      if (proxy_needs_restart(mp)) {
+        log_info(LD_GENERAL, "Preparing managed proxy '%s' for restart.",
+                 mp->argv[0]);
+        proxy_prepare_for_restart(mp);
+      } else { /* it doesn't need to be restarted. */
+        log_info(LD_GENERAL, "Nothing changed for managed proxy '%s' after "
+                 "HUP: not restarting.", mp->argv[0]);
+      }
+
+      continue;
+    }
+
+    /* If the proxy is not fully configured, try to configure it
+       further. */
+    if (!proxy_configuration_finished(mp))
+      if (configure_proxy(mp) == 1)
+        at_least_a_proxy_config_finished = 1;
+
+  } SMARTLIST_FOREACH_END(mp);
+
+  smartlist_free(tmp);
+  check_if_restarts_needed = 0;
+  assert_unconfigured_count_ok();
+
+  if (at_least_a_proxy_config_finished)
+    mark_my_descriptor_dirty("configured managed proxies");
+}
+
+/** Attempt to continue configuring managed proxy <b>mp</b>.
+ *  Return 1 if the transport configuration finished, and return 0
+ *  otherwise (if we still have more configuring to do for this
+ *  proxy). */
+STATIC int
+configure_proxy(managed_proxy_t *mp)
+{
+  int configuration_finished = 0;
+  smartlist_t *proxy_output = NULL;
+  enum stream_status stream_status = 0;
+
+  /* if we haven't launched the proxy yet, do it now */
+  if (mp->conf_state == PT_PROTO_INFANT) {
+    if (launch_managed_proxy(mp) < 0) { /* launch fail */
+      mp->conf_state = PT_PROTO_FAILED_LAUNCH;
+      handle_finished_proxy(mp);
+    }
+    return 0;
+  }
+
+  tor_assert(mp->conf_state != PT_PROTO_INFANT);
+  tor_assert(mp->process_handle);
+
+  proxy_output =
+    tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle),
+                              &stream_status);
+  if (!proxy_output) { /* failed to get input from proxy */
+    if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */
+      mp->conf_state = PT_PROTO_BROKEN;
+      log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' "
+               "is '%s'. Most probably the managed proxy stopped running. "
+               "This might be a bug of the managed proxy, a bug of Tor, or "
+               "a misconfiguration. Please enable logging on your managed "
+               "proxy and check the logs for errors.",
+               mp->argv[0], stream_status_to_string(stream_status));
+    }
+
+    goto done;
+  }
+
+  /* Handle lines. */
+  SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) {
+    handle_proxy_line(line, mp);
+    if (proxy_configuration_finished(mp))
+      goto done;
+  } SMARTLIST_FOREACH_END(line);
+
+ done:
+  /* if the proxy finished configuring, exit the loop. */
+  if (proxy_configuration_finished(mp)) {
+    handle_finished_proxy(mp);
+    configuration_finished = 1;
+  }
+
+  if (proxy_output) {
+    SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp));
+    smartlist_free(proxy_output);
+  }
+
+  return configuration_finished;
+}
+
+/** Register server managed proxy <b>mp</b> transports to state */
+static void
+register_server_proxy(const managed_proxy_t *mp)
+{
+  tor_assert(mp->conf_state != PT_PROTO_COMPLETED);
+
+  SMARTLIST_FOREACH_BEGIN(mp->transports, transport_t *, t) {
+    save_transport_to_state(t->name, &t->addr, t->port);
+    log_notice(LD_GENERAL, "Registered server transport '%s' at '%s'",
+               t->name, fmt_addrport(&t->addr, t->port));
+    control_event_transport_launched("server", t->name, &t->addr, t->port);
+  } SMARTLIST_FOREACH_END(t);
+}
+
+/** Register all the transports supported by client managed proxy
+ *  <b>mp</b> to the bridge subsystem. */
+static void
+register_client_proxy(const managed_proxy_t *mp)
+{
+  int r;
+
+  tor_assert(mp->conf_state != PT_PROTO_COMPLETED);
+
+  SMARTLIST_FOREACH_BEGIN(mp->transports, transport_t *, t) {
+    transport_t *transport_tmp = transport_copy(t);
+    r = transport_add(transport_tmp);
+    switch (r) {
+    case -1:
+      log_notice(LD_GENERAL, "Could not add transport %s. Skipping.", t->name);
+      transport_free(transport_tmp);
+      break;
+    case 0:
+      log_info(LD_GENERAL, "Successfully registered transport %s", t->name);
+      control_event_transport_launched("client", t->name, &t->addr, t->port);
+      break;
+    case 1:
+      log_info(LD_GENERAL, "Successfully registered transport %s", t->name);
+      control_event_transport_launched("client", t->name, &t->addr, t->port);
+      transport_free(transport_tmp);
+      break;
+    }
+  } SMARTLIST_FOREACH_END(t);
+}
+
+/** Register the transports of managed proxy <b>mp</b>. */
+static inline void
+register_proxy(const managed_proxy_t *mp)
+{
+  if (mp->is_server)
+    register_server_proxy(mp);
+  else
+    register_client_proxy(mp);
+}
+
+/** Free memory allocated by managed proxy <b>mp</b>. */
+STATIC void
+managed_proxy_destroy(managed_proxy_t *mp,
+                      int also_terminate_process)
+{
+  SMARTLIST_FOREACH(mp->transports, transport_t *, t, transport_free(t));
+
+  /* free the transports smartlist */
+  smartlist_free(mp->transports);
+
+  /* free the transports_to_launch smartlist */
+  SMARTLIST_FOREACH(mp->transports_to_launch, char *, t, tor_free(t));
+  smartlist_free(mp->transports_to_launch);
+
+  /* remove it from the list of managed proxies */
+  if (managed_proxy_list)
+    smartlist_remove(managed_proxy_list, mp);
+
+  /* free the argv */
+  free_execve_args(mp->argv);
+
+  /* free the outgoing proxy URI */
+  tor_free(mp->proxy_uri);
+
+  tor_process_handle_destroy(mp->process_handle, also_terminate_process);
+  mp->process_handle = NULL;
+
+  tor_free(mp);
+}
+
+/** Convert the tor proxy options to a URI suitable for TOR_PT_PROXY.
+ * Return a newly allocated string containing the URI, or NULL if no
+ * proxy is set. */
+STATIC char *
+get_pt_proxy_uri(void)
+{
+  const or_options_t *options = get_options();
+  char *uri = NULL;
+
+  if (options->Socks4Proxy || options->Socks5Proxy || options->HTTPSProxy) {
+    char addr[TOR_ADDR_BUF_LEN+1];
+
+    if (options->Socks4Proxy) {
+      tor_addr_to_str(addr, &options->Socks4ProxyAddr, sizeof(addr), 1);
+      tor_asprintf(&uri, "socks4a://%s:%d", addr, options->Socks4ProxyPort);
+    } else if (options->Socks5Proxy) {
+      tor_addr_to_str(addr, &options->Socks5ProxyAddr, sizeof(addr), 1);
+      if (!options->Socks5ProxyUsername && !options->Socks5ProxyPassword) {
+        tor_asprintf(&uri, "socks5://%s:%d", addr, options->Socks5ProxyPort);
+      } else {
+        tor_asprintf(&uri, "socks5://%s:%s@%s:%d",
+                     options->Socks5ProxyUsername,
+                     options->Socks5ProxyPassword,
+                     addr, options->Socks5ProxyPort);
+      }
+    } else if (options->HTTPSProxy) {
+      tor_addr_to_str(addr, &options->HTTPSProxyAddr, sizeof(addr), 1);
+      if (!options->HTTPSProxyAuthenticator) {
+        tor_asprintf(&uri, "http://%s:%d", addr, options->HTTPSProxyPort);
+      } else {
+        tor_asprintf(&uri, "http://%s@%s:%d", options->HTTPSProxyAuthenticator,
+                     addr, options->HTTPSProxyPort);
+      }
+    }
+  }
+
+  return uri;
+}
+
+/** Handle a configured or broken managed proxy <b>mp</b>. */
+static void
+handle_finished_proxy(managed_proxy_t *mp)
+{
+  switch (mp->conf_state) {
+  case PT_PROTO_BROKEN: /* if broken: */
+    managed_proxy_destroy(mp, 1); /* annihilate it. */
+    break;
+  case PT_PROTO_FAILED_LAUNCH: /* if it failed before launching: */
+    managed_proxy_destroy(mp, 0); /* destroy it but don't terminate */
+    break;
+  case PT_PROTO_CONFIGURED: /* if configured correctly: */
+    if (mp->proxy_uri && !mp->proxy_supported) {
+      log_warn(LD_CONFIG, "Managed proxy '%s' did not configure the "
+               "specified outgoing proxy and will be terminated.",
+               mp->argv[0]);
+      managed_proxy_destroy(mp, 1); /* annihilate it. */
+      break;
+    }
+    register_proxy(mp); /* register its transports */
+    mp->conf_state = PT_PROTO_COMPLETED; /* and mark it as completed. */
+    break;
+  case PT_PROTO_INFANT:
+  case PT_PROTO_LAUNCHED:
+  case PT_PROTO_ACCEPTING_METHODS:
+  case PT_PROTO_COMPLETED:
+  default:
+    log_warn(LD_CONFIG, "Unexpected state '%d' of managed proxy '%s'.",
+             (int)mp->conf_state, mp->argv[0]);
+    tor_assert(0);
+  }
+
+  unconfigured_proxies_n--;
+}
+
+/** Return true if the configuration of the managed proxy <b>mp</b> is
+    finished. */
+static inline int
+proxy_configuration_finished(const managed_proxy_t *mp)
+{
+  return (mp->conf_state == PT_PROTO_CONFIGURED ||
+          mp->conf_state == PT_PROTO_BROKEN ||
+          mp->conf_state == PT_PROTO_FAILED_LAUNCH);
+}
+
+/** This function is called when a proxy sends an {S,C}METHODS DONE message. */
+static void
+handle_methods_done(const managed_proxy_t *mp)
+{
+  tor_assert(mp->transports);
+
+  if (smartlist_len(mp->transports) == 0)
+    log_notice(LD_GENERAL, "Managed proxy '%s' was spawned successfully, "
+               "but it didn't launch any pluggable transport listeners!",
+               mp->argv[0]);
+
+  log_info(LD_CONFIG, "%s managed proxy '%s' configuration completed!",
+           mp->is_server ? "Server" : "Client",
+           mp->argv[0]);
+}
+
+/** Handle a configuration protocol <b>line</b> received from a
+ *  managed proxy <b>mp</b>. */
+STATIC void
+handle_proxy_line(const char *line, managed_proxy_t *mp)
+{
+  log_info(LD_GENERAL, "Got a line from managed proxy '%s': (%s)",
+           mp->argv[0], line);
+
+  if (!strcmpstart(line, PROTO_ENV_ERROR)) {
+    if (mp->conf_state != PT_PROTO_LAUNCHED)
+      goto err;
+
+    parse_env_error(line);
+    goto err;
+  } else if (!strcmpstart(line, PROTO_NEG_FAIL)) {
+    if (mp->conf_state != PT_PROTO_LAUNCHED)
+      goto err;
+
+    log_warn(LD_CONFIG, "Managed proxy could not pick a "
+             "configuration protocol version.");
+    goto err;
+  } else if (!strcmpstart(line, PROTO_NEG_SUCCESS)) {
+    if (mp->conf_state != PT_PROTO_LAUNCHED)
+      goto err;
+
+    if (parse_version(line,mp) < 0)
+      goto err;
+
+    tor_assert(mp->conf_protocol != 0);
+    mp->conf_state = PT_PROTO_ACCEPTING_METHODS;
+    return;
+  } else if (!strcmpstart(line, PROTO_CMETHODS_DONE)) {
+    if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS)
+      goto err;
+
+    handle_methods_done(mp);
+
+    mp->conf_state = PT_PROTO_CONFIGURED;
+    return;
+  } else if (!strcmpstart(line, PROTO_SMETHODS_DONE)) {
+    if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS)
+      goto err;
+
+    handle_methods_done(mp);
+
+    mp->conf_state = PT_PROTO_CONFIGURED;
+    return;
+  } else if (!strcmpstart(line, PROTO_CMETHOD_ERROR)) {
+    if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS)
+      goto err;
+
+    parse_client_method_error(line);
+    goto err;
+  } else if (!strcmpstart(line, PROTO_SMETHOD_ERROR)) {
+    if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS)
+      goto err;
+
+    parse_server_method_error(line);
+    goto err;
+  } else if (!strcmpstart(line, PROTO_CMETHOD)) {
+    if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS)
+      goto err;
+
+    if (parse_cmethod_line(line, mp) < 0)
+      goto err;
+
+    return;
+  } else if (!strcmpstart(line, PROTO_SMETHOD)) {
+    if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS)
+      goto err;
+
+    if (parse_smethod_line(line, mp) < 0)
+      goto err;
+
+    return;
+  } else if (!strcmpstart(line, PROTO_PROXY_DONE)) {
+    if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS)
+      goto err;
+
+    if (mp->proxy_uri) {
+      mp->proxy_supported = 1;
+      return;
+    }
+
+    /* No proxy was configured, this should log */
+  } else if (!strcmpstart(line, PROTO_PROXY_ERROR)) {
+    if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS)
+      goto err;
+
+    parse_proxy_error(line);
+    goto err;
+  } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) {
+    /* managed proxy launch failed: parse error message to learn why. */
+    int retval, child_state, saved_errno;
+    retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x",
+                        &child_state, &saved_errno);
+    if (retval == 2) {
+      log_warn(LD_GENERAL,
+               "Could not launch managed proxy executable at '%s' ('%s').",
+               mp->argv[0], strerror(saved_errno));
+    } else { /* failed to parse error message */
+      log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.",
+               mp->argv[0]);
+    }
+
+    mp->conf_state = PT_PROTO_FAILED_LAUNCH;
+    return;
+  }
+
+  log_notice(LD_GENERAL, "Unknown line received by managed proxy (%s).", line);
+  return;
+
+ err:
+  mp->conf_state = PT_PROTO_BROKEN;
+  log_warn(LD_CONFIG, "Managed proxy at '%s' failed the configuration protocol"
+           " and will be destroyed.", mp->argv[0]);
+}
+
+/** Parses an ENV-ERROR <b>line</b> and warns the user accordingly. */
+STATIC void
+parse_env_error(const char *line)
+{
+  /* (Length of the protocol string) plus (a space) and (the first char of
+     the error message) */
+  if (strlen(line) < (strlen(PROTO_ENV_ERROR) + 2))
+    log_notice(LD_CONFIG, "Managed proxy sent us an %s without an error "
+               "message.", PROTO_ENV_ERROR);
+
+  log_warn(LD_CONFIG, "Managed proxy couldn't understand the "
+           "pluggable transport environment variables. (%s)",
+           line+strlen(PROTO_ENV_ERROR)+1);
+}
+
+/** Handles a VERSION <b>line</b>. Updates the configuration protocol
+ *  version in <b>mp</b>. */
+STATIC int
+parse_version(const char *line, managed_proxy_t *mp)
+{
+  if (strlen(line) < (strlen(PROTO_NEG_SUCCESS) + 2)) {
+    log_warn(LD_CONFIG, "Managed proxy sent us malformed %s line.",
+             PROTO_NEG_SUCCESS);
+    return -1;
+  }
+
+  if (strcmp("1", line+strlen(PROTO_NEG_SUCCESS)+1)) { /* hardcoded temp */
+    log_warn(LD_CONFIG, "Managed proxy tried to negotiate on version '%s'. "
+             "We only support version '1'", line+strlen(PROTO_NEG_SUCCESS)+1);
+    return -1;
+  }
+
+  mp->conf_protocol = PROTO_VERSION_ONE; /* temp. till more versions appear */
+  return 0;
+}
+
+/** Parses {C,S}METHOD-ERROR <b>line</b> and warns the user
+ *  accordingly.  If <b>is_server</b> it is an SMETHOD-ERROR,
+ *  otherwise it is a CMETHOD-ERROR. */
+static void
+parse_method_error(const char *line, int is_server)
+{
+  const char* error = is_server ?
+    PROTO_SMETHOD_ERROR : PROTO_CMETHOD_ERROR;
+
+  /* (Length of the protocol string) plus (a space) and (the first char of
+     the error message) */
+  if (strlen(line) < (strlen(error) + 2))
+    log_warn(LD_CONFIG, "Managed proxy sent us an %s without an error "
+             "message.", error);
+
+  log_warn(LD_CONFIG, "%s managed proxy encountered a method error. (%s)",
+           is_server ? "Server" : "Client",
+           line+strlen(error)+1);
+}
+
+/** A helper for parse_{c,s}method_line(), bootstraps its
+ *  functionalities. If <b>is_smethod</b> is true then the
+ *  the line to parse is a SMETHOD line otherwise it is a
+ *  CMETHOD line*/
+static int
+parse_method_line_helper(const char *line,
+                         managed_proxy_t *mp,
+                         int is_smethod)
+{
+  int item_index = 0;
+  int r;
+
+  char *transport_name=NULL;
+  char *args_string=NULL;
+  char *addrport=NULL;
+  int socks_ver=PROXY_NONE;
+  char *address=NULL;
+  uint16_t port = 0;
+
+  const char *method_str = is_smethod ? PROTO_SMETHOD : PROTO_CMETHOD;
+  const int min_args_count = is_smethod ? 3 : 4;
+
+  tor_addr_t tor_addr;
+  transport_t *transport=NULL;
+  smartlist_t *items= smartlist_new();
+
+  smartlist_split_string(items, line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  if (smartlist_len(items) < min_args_count) {
+    log_warn(LD_CONFIG, "Managed proxy sent us a %s line "
+             "with too few arguments.", method_str);
+    goto err;
+  }
+
+  tor_assert(!strcmp(smartlist_get(items, item_index),method_str));
+  ++item_index;
+
+  transport_name = smartlist_get(items,item_index);
+  ++item_index;
+  if (!string_is_C_identifier(transport_name)) {
+    log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).",
+             transport_name);
+    goto err;
+  }
+
+  /** Check for the proxy method sent to us in CMETHOD line. */
+  if (!is_smethod) {
+    const char *socks_ver_str = smartlist_get(items,item_index);
+    ++item_index;
+
+    if (!strcmp(socks_ver_str,"socks4")) {
+      socks_ver = PROXY_SOCKS4;
+    } else if (!strcmp(socks_ver_str,"socks5")) {
+      socks_ver = PROXY_SOCKS5;
+    } else {
+      log_warn(LD_CONFIG, "Client managed proxy sent us a proxy protocol "
+               "we don't recognize. (%s)", socks_ver_str);
+      goto err;
+    }
+  }
+
+  addrport = smartlist_get(items, item_index);
+  ++item_index;
+  if (tor_addr_port_split(LOG_WARN, addrport, &address, &port)<0) {
+    log_warn(LD_CONFIG, "Error parsing transport address '%s'", addrport);
+    goto err;
+  }
+
+  if (!port) {
+    log_warn(LD_CONFIG,
+             "Transport address '%s' has no port.", addrport);
+    goto err;
+  }
+
+  if (tor_addr_parse(&tor_addr, address) < 0) {
+    log_warn(LD_CONFIG, "Error parsing transport address '%s'", address);
+    goto err;
+  }
+
+  /** Check for options in the SMETHOD line. */
+  if (is_smethod && smartlist_len(items) > min_args_count) {
+    /* Seems like there are also some [options] in the SMETHOD line.
+       Let's see if we can parse them. */
+    char *options_string = smartlist_get(items, item_index);
+    log_debug(LD_CONFIG, "Got options_string: %s", options_string);
+    if (!strcmpstart(options_string, "ARGS:")) {
+      args_string = options_string+strlen("ARGS:");
+      log_debug(LD_CONFIG, "Got ARGS: %s", args_string);
+    }
+  }
+
+  transport = transport_new(&tor_addr, port, transport_name,
+                            socks_ver, args_string);
+
+  smartlist_add(mp->transports, transport);
+
+  /** Logs info about line parsing success for client or server */
+  if (is_smethod) {
+    log_info(LD_CONFIG, "Server transport %s at %s:%d.",
+             transport_name, address, (int)port);
+  } else {
+    log_info(LD_CONFIG, "Transport %s at %s:%d with SOCKS %d. "
+             "Attached to managed proxy.",
+             transport_name, address, (int)port, socks_ver);
+  }
+
+  r=0;
+  goto done;
+
+ err:
+  r = -1;
+
+ done:
+  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
+  smartlist_free(items);
+  tor_free(address);
+  return r;
+}
+
+/** Parses an SMETHOD <b>line</b> and if well-formed it registers the
+ *  new transport in <b>mp</b>. */
+STATIC int
+parse_smethod_line(const char *line, managed_proxy_t *mp)
+{
+  /* Example of legit SMETHOD line:
+     SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */
+  return parse_method_line_helper(line, mp, 1);
+}
+
+/** Parses a CMETHOD <b>line</b>, and if well-formed it registers
+ *  the new transport in <b>mp</b>. */
+STATIC int
+parse_cmethod_line(const char *line, managed_proxy_t *mp)
+{
+  /* Example of legit CMETHOD line:
+     CMETHOD obfs2 socks5 127.0.0.1:35713 */
+  return parse_method_line_helper(line, mp, 0);
+}
+
+/** Parses an PROXY-ERROR <b>line</b> and warns the user accordingly. */
+STATIC void
+parse_proxy_error(const char *line)
+{
+  /* (Length of the protocol string) plus (a space) and (the first char of
+     the error message) */
+  if (strlen(line) < (strlen(PROTO_PROXY_ERROR) + 2))
+    log_notice(LD_CONFIG, "Managed proxy sent us an %s without an error "
+               "message.", PROTO_PROXY_ERROR);
+
+  log_warn(LD_CONFIG, "Managed proxy failed to configure the "
+           "pluggable transport's outgoing proxy. (%s)",
+           line+strlen(PROTO_PROXY_ERROR)+1);
+}
+
+/** Return a newly allocated string that tor should place in
+ * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server
+ * manged proxy in <b>mp</b>. Return NULL if no such options are found. */
+STATIC char *
+get_transport_options_for_server_proxy(const managed_proxy_t *mp)
+{
+  char *options_string = NULL;
+  smartlist_t *string_sl = smartlist_new();
+
+  tor_assert(mp->is_server);
+
+  /** Loop over the transports of the proxy. If we have options for
+      any of them, format them appropriately and place them in our
+      smartlist. Finally, join our smartlist to get the final
+      string. */
+  SMARTLIST_FOREACH_BEGIN(mp->transports_to_launch, const char *, transport) {
+    smartlist_t *options_tmp_sl = NULL;
+    options_tmp_sl = get_options_for_server_transport(transport);
+    if (!options_tmp_sl)
+      continue;
+
+    /** Loop over the options of this transport, escape them, and
+        place them in the smartlist. */
+    SMARTLIST_FOREACH_BEGIN(options_tmp_sl, const char *, options) {
+      char *escaped_opts = tor_escape_str_for_pt_args(options, ":;\\");
+      smartlist_add_asprintf(string_sl, "%s:%s",
+                             transport, escaped_opts);
+      tor_free(escaped_opts);
+    } SMARTLIST_FOREACH_END(options);
+
+    SMARTLIST_FOREACH(options_tmp_sl, char *, c, tor_free(c));
+    smartlist_free(options_tmp_sl);
+  } SMARTLIST_FOREACH_END(transport);
+
+  if (smartlist_len(string_sl)) {
+    options_string = smartlist_join_strings(string_sl, ";", 0, NULL);
+  }
+
+  SMARTLIST_FOREACH(string_sl, char *, t, tor_free(t));
+  smartlist_free(string_sl);
+
+  return options_string;
+}
+
+/** Return the string that tor should place in TOR_PT_SERVER_BINDADDR
+ *  while configuring the server managed proxy in <b>mp</b>. The
+ *  string is stored in the heap, and it's the responsibility of
+ *  the caller to deallocate it after its use. */
+static char *
+get_bindaddr_for_server_proxy(const managed_proxy_t *mp)
+{
+  char *bindaddr_result = NULL;
+  char *bindaddr_tmp = NULL;
+  smartlist_t *string_tmp = smartlist_new();
+
+  tor_assert(mp->is_server);
+
+  SMARTLIST_FOREACH_BEGIN(mp->transports_to_launch, char *, t) {
+    bindaddr_tmp = get_stored_bindaddr_for_server_transport(t);
+
+    smartlist_add_asprintf(string_tmp, "%s-%s", t, bindaddr_tmp);
+
+    tor_free(bindaddr_tmp);
+  } SMARTLIST_FOREACH_END(t);
+
+  bindaddr_result = smartlist_join_strings(string_tmp, ",", 0, NULL);
+
+  SMARTLIST_FOREACH(string_tmp, char *, t, tor_free(t));
+  smartlist_free(string_tmp);
+
+  return bindaddr_result;
+}
+
+/** Return a newly allocated process_environment_t * for <b>mp</b>'s
+ * process. */
+static process_environment_t *
+create_managed_proxy_environment(const managed_proxy_t *mp)
+{
+  const or_options_t *options = get_options();
+
+  /* Environment variables to be added to or set in mp's environment. */
+  smartlist_t *envs = smartlist_new();
+  /* XXXX The next time someone touches this code, shorten the name of
+   * set_environment_variable_in_smartlist, add a
+   * set_env_var_in_smartlist_asprintf function, and get rid of the
+   * silly extra envs smartlist. */
+
+  /* The final environment to be passed to mp. */
+  smartlist_t *merged_env_vars = get_current_process_environment_variables();
+
+  process_environment_t *env;
+
+  {
+    char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */
+    smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp);
+    tor_free(state_tmp);
+  }
+
+  smartlist_add_strdup(envs, "TOR_PT_MANAGED_TRANSPORT_VER=1");
+
+  {
+    char *transports_to_launch =
+      smartlist_join_strings(mp->transports_to_launch, ",", 0, NULL);
+
+    smartlist_add_asprintf(envs,
+                           mp->is_server ?
+                           "TOR_PT_SERVER_TRANSPORTS=%s" :
+                           "TOR_PT_CLIENT_TRANSPORTS=%s",
+                           transports_to_launch);
+
+    tor_free(transports_to_launch);
+  }
+
+  if (mp->is_server) {
+    {
+      char *orport_tmp =
+        get_first_listener_addrport_string(CONN_TYPE_OR_LISTENER);
+      if (orport_tmp) {
+        smartlist_add_asprintf(envs, "TOR_PT_ORPORT=%s", orport_tmp);
+        tor_free(orport_tmp);
+      }
+    }
+
+    {
+      char *bindaddr_tmp = get_bindaddr_for_server_proxy(mp);
+      smartlist_add_asprintf(envs, "TOR_PT_SERVER_BINDADDR=%s", bindaddr_tmp);
+      tor_free(bindaddr_tmp);
+    }
+
+    {
+      char *server_transport_options =
+        get_transport_options_for_server_proxy(mp);
+      if (server_transport_options) {
+        smartlist_add_asprintf(envs, "TOR_PT_SERVER_TRANSPORT_OPTIONS=%s",
+                               server_transport_options);
+        tor_free(server_transport_options);
+      }
+    }
+
+    /* XXXX Remove the '=' here once versions of obfsproxy which
+     * assert that this env var exists are sufficiently dead.
+     *
+     * (If we remove this line entirely, some joker will stick this
+     * variable in Tor's environment and crash PTs that try to parse
+     * it even when not run in server mode.) */
+
+    if (options->ExtORPort_lines) {
+      char *ext_or_addrport_tmp =
+        get_first_listener_addrport_string(CONN_TYPE_EXT_OR_LISTENER);
+      char *cookie_file_loc = get_ext_or_auth_cookie_file_name();
+
+      if (ext_or_addrport_tmp) {
+        smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=%s",
+                               ext_or_addrport_tmp);
+      }
+      smartlist_add_asprintf(envs, "TOR_PT_AUTH_COOKIE_FILE=%s",
+                             cookie_file_loc);
+
+      tor_free(ext_or_addrport_tmp);
+      tor_free(cookie_file_loc);
+
+    } else {
+      smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=");
+    }
+
+    /* All new versions of tor will keep stdin open, so PTs can use it
+     * as a reliable termination detection mechanism.
+     */
+    smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1");
+  } else {
+    /* If ClientTransportPlugin has a HTTPS/SOCKS proxy configured, set the
+     * TOR_PT_PROXY line.
+     */
+
+    if (mp->proxy_uri) {
+      smartlist_add_asprintf(envs, "TOR_PT_PROXY=%s", mp->proxy_uri);
+    }
+  }
+
+  SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) {
+    set_environment_variable_in_smartlist(merged_env_vars, env_var,
+                                          tor_free_, 1);
+  } SMARTLIST_FOREACH_END(env_var);
+
+  env = process_environment_make(merged_env_vars);
+
+  smartlist_free(envs);
+
+  SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x));
+  smartlist_free(merged_env_vars);
+
+  return env;
+}
+
+/** Create and return a new managed proxy for <b>transport</b> using
+ *  <b>proxy_argv</b>.  Also, add it to the global managed proxy list. If
+ *  <b>is_server</b> is true, it's a server managed proxy.  Takes ownership of
+ *  <b>proxy_argv</b>.
+ *
+ * Requires that proxy_argv have at least one element. */
+STATIC managed_proxy_t *
+managed_proxy_create(const smartlist_t *with_transport_list,
+                     char **proxy_argv, int is_server)
+{
+  managed_proxy_t *mp = tor_malloc_zero(sizeof(managed_proxy_t));
+  mp->conf_state = PT_PROTO_INFANT;
+  mp->is_server = is_server;
+  mp->argv = proxy_argv;
+  mp->transports = smartlist_new();
+  mp->proxy_uri = get_pt_proxy_uri();
+
+  mp->transports_to_launch = smartlist_new();
+  SMARTLIST_FOREACH(with_transport_list, const char *, transport,
+                    add_transport_to_proxy(transport, mp));
+
+  /* register the managed proxy */
+  if (!managed_proxy_list)
+    managed_proxy_list = smartlist_new();
+  smartlist_add(managed_proxy_list, mp);
+  unconfigured_proxies_n++;
+
+  assert_unconfigured_count_ok();
+
+  return mp;
+}
+
+/** Register proxy with <b>proxy_argv</b>, supporting transports in
+ *  <b>transport_list</b>, to the managed proxy subsystem.
+ *  If <b>is_server</b> is true, then the proxy is a server proxy.
+ *
+ * Takes ownership of proxy_argv.
+ *
+ * Requires that proxy_argv be a NULL-terminated array of command-line
+ * elements, containing at least one element.
+ **/
+MOCK_IMPL(void,
+pt_kickstart_proxy, (const smartlist_t *with_transport_list,
+                     char **proxy_argv, int is_server))
+{
+  managed_proxy_t *mp=NULL;
+  transport_t *old_transport = NULL;
+
+  if (!proxy_argv || !proxy_argv[0]) {
+    return;
+  }
+
+  mp = get_managed_proxy_by_argv_and_type(proxy_argv, is_server);
+
+  if (!mp) { /* we haven't seen this proxy before */
+    managed_proxy_create(with_transport_list, proxy_argv, is_server);
+
+  } else { /* known proxy. add its transport to its transport list */
+    if (mp->was_around_before_config_read) {
+      /* If this managed proxy was around even before we read the
+         config this time, it means that it was already enabled before
+         and is not useless and should be kept. If it's marked for
+         removal, unmark it and make sure that we check whether it
+         needs to be restarted. */
+      if (mp->marked_for_removal) {
+        mp->marked_for_removal = 0;
+        check_if_restarts_needed = 1;
+      }
+
+      /* For each new transport, check if the managed proxy used to
+         support it before the SIGHUP. If that was the case, make sure
+         it doesn't get removed because we might reuse it. */
+      SMARTLIST_FOREACH_BEGIN(with_transport_list, const char *, transport) {
+        old_transport = transport_get_by_name(transport);
+        if (old_transport)
+          old_transport->marked_for_removal = 0;
+      } SMARTLIST_FOREACH_END(transport);
+    }
+
+    SMARTLIST_FOREACH(with_transport_list, const char *, transport,
+                      add_transport_to_proxy(transport, mp));
+    free_execve_args(proxy_argv);
+  }
+}
+
+/** Frees the array of pointers in <b>arg</b> used as arguments to
+    execve(2). */
+STATIC void
+free_execve_args(char **arg)
+{
+  char **tmp = arg;
+  while (*tmp) /* use the fact that the last element of the array is a
+                  NULL pointer to know when to stop freeing */
+    tor_free_(*tmp++);
+
+  tor_free(arg);
+}
+
+/** Tor will read its config.
+ *  Prepare the managed proxy list so that proxies not used in the new
+ *  config will shutdown, and proxies that need to spawn different
+ *  transports will do so. */
+void
+pt_prepare_proxy_list_for_config_read(void)
+{
+  if (!managed_proxy_list)
+    return;
+
+  assert_unconfigured_count_ok();
+  SMARTLIST_FOREACH_BEGIN(managed_proxy_list, managed_proxy_t *, mp) {
+    /* Destroy unconfigured proxies. */
+    if (mp->conf_state != PT_PROTO_COMPLETED) {
+      SMARTLIST_DEL_CURRENT(managed_proxy_list, mp);
+      managed_proxy_destroy(mp, 1);
+      unconfigured_proxies_n--;
+      continue;
+    }
+
+    tor_assert(mp->conf_state == PT_PROTO_COMPLETED);
+
+    /* Mark all proxies for removal, and also note that they have been
+       here before the config read. */
+    mp->marked_for_removal = 1;
+    mp->was_around_before_config_read = 1;
+    SMARTLIST_FOREACH(mp->transports_to_launch, char *, t, tor_free(t));
+    smartlist_clear(mp->transports_to_launch);
+  } SMARTLIST_FOREACH_END(mp);
+
+  assert_unconfigured_count_ok();
+
+  tor_assert(unconfigured_proxies_n == 0);
+}
+
+/** Return a smartlist containing the ports where our pluggable
+ *  transports are listening. */
+smartlist_t *
+get_transport_proxy_ports(void)
+{
+  smartlist_t *sl = NULL;
+
+  if (!managed_proxy_list)
+    return NULL;
+
+  /** XXX assume that external proxy ports have been forwarded
+      manually */
+  SMARTLIST_FOREACH_BEGIN(managed_proxy_list, const managed_proxy_t *, mp) {
+    if (!mp->is_server || mp->conf_state != PT_PROTO_COMPLETED)
+      continue;
+
+    if (!sl) sl = smartlist_new();
+
+    tor_assert(mp->transports);
+    SMARTLIST_FOREACH(mp->transports, const transport_t *, t,
+                      smartlist_add_asprintf(sl, "%u:%u", t->port, t->port));
+
+  } SMARTLIST_FOREACH_END(mp);
+
+  return sl;
+}
+
+/** Return the pluggable transport string that we should display in
+ *  our extra-info descriptor. If we shouldn't display such a string,
+ *  or we have nothing to display, return NULL. The string is
+ *  allocated on the heap and it's the responsibility of the caller to
+ *  free it. */
+char *
+pt_get_extra_info_descriptor_string(void)
+{
+  char *the_string = NULL;
+  smartlist_t *string_chunks = NULL;
+
+  if (!managed_proxy_list)
+    return NULL;
+
+  string_chunks = smartlist_new();
+
+  /* For each managed proxy, add its transports to the chunks list. */
+  SMARTLIST_FOREACH_BEGIN(managed_proxy_list,  const managed_proxy_t *, mp) {
+    if ((!mp->is_server) || (mp->conf_state != PT_PROTO_COMPLETED))
+      continue;
+
+    tor_assert(mp->transports);
+
+    SMARTLIST_FOREACH_BEGIN(mp->transports, const transport_t *, t) {
+      char *transport_args = NULL;
+
+      /* If the transport proxy returned "0.0.0.0" as its address, and
+       * we know our external IP address, use it. Otherwise, use the
+       * returned address. */
+      const char *addrport = NULL;
+      uint32_t external_ip_address = 0;
+      if (tor_addr_is_null(&t->addr) &&
+          router_pick_published_address(get_options(),
+                                        &external_ip_address, 0) >= 0) {
+        tor_addr_t addr;
+        tor_addr_from_ipv4h(&addr, external_ip_address);
+        addrport = fmt_addrport(&addr, t->port);
+      } else {
+        addrport = fmt_addrport(&t->addr, t->port);
+      }
+
+      /* If this transport has any arguments with it, prepend a space
+         to them so that we can add them to the transport line. */
+      if (t->extra_info_args)
+        tor_asprintf(&transport_args, " %s", t->extra_info_args);
+
+      smartlist_add_asprintf(string_chunks,
+                             "transport %s %s%s",
+                             t->name, addrport,
+                             transport_args ? transport_args : "");
+      tor_free(transport_args);
+    } SMARTLIST_FOREACH_END(t);
+
+  } SMARTLIST_FOREACH_END(mp);
+
+  if (smartlist_len(string_chunks) == 0) {
+    smartlist_free(string_chunks);
+    return NULL;
+  }
+
+  /* Join all the chunks into the final string. */
+  the_string = smartlist_join_strings(string_chunks, "\n", 1, NULL);
+
+  SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s));
+  smartlist_free(string_chunks);
+
+  return the_string;
+}
+
+/** Stringify the SOCKS arguments in <b>socks_args</b> according to
+ *  180_pluggable_transport.txt.  The string is allocated on the heap
+ *  and it's the responsibility of the caller to free it after use. */
+char *
+pt_stringify_socks_args(const smartlist_t *socks_args)
+{
+  /* tmp place to store escaped socks arguments, so that we can
+     concatenate them up afterwards */
+  smartlist_t *sl_tmp = NULL;
+  char *escaped_string = NULL;
+  char *new_string = NULL;
+
+  tor_assert(socks_args);
+  tor_assert(smartlist_len(socks_args) > 0);
+
+  sl_tmp = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(socks_args, const char *, s) {
+    /* Escape ';' and '\'. */
+    escaped_string = tor_escape_str_for_pt_args(s, ";\\");
+    if (!escaped_string)
+      goto done;
+
+    smartlist_add(sl_tmp, escaped_string);
+  } SMARTLIST_FOREACH_END(s);
+
+  new_string = smartlist_join_strings(sl_tmp, ";", 0, NULL);
+
+ done:
+  SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s));
+  smartlist_free(sl_tmp);
+
+  return new_string;
+}
+
+/** Return a string of the SOCKS arguments that we should pass to the
+ *  pluggable transports proxy in <b>addr</b>:<b>port</b> according to
+ *  180_pluggable_transport.txt.  The string is allocated on the heap
+ *  and it's the responsibility of the caller to free it after use. */
+char *
+pt_get_socks_args_for_proxy_addrport(const tor_addr_t *addr, uint16_t port)
+{
+  const smartlist_t *socks_args = NULL;
+
+  socks_args = get_socks_args_by_bridge_addrport(addr, port);
+  if (!socks_args)
+    return NULL;
+
+  return pt_stringify_socks_args(socks_args);
+}
+
+/** The tor config was read.
+ *  Destroy all managed proxies that were marked by a previous call to
+ *  prepare_proxy_list_for_config_read() and are not used by the new
+ *  config. */
+void
+sweep_proxy_list(void)
+{
+  if (!managed_proxy_list)
+    return;
+  assert_unconfigured_count_ok();
+  SMARTLIST_FOREACH_BEGIN(managed_proxy_list, managed_proxy_t *, mp) {
+    if (mp->marked_for_removal) {
+      SMARTLIST_DEL_CURRENT(managed_proxy_list, mp);
+      managed_proxy_destroy(mp, 1);
+    }
+  } SMARTLIST_FOREACH_END(mp);
+  assert_unconfigured_count_ok();
+}
+
+/** Release all storage held by the pluggable transports subsystem. */
+void
+pt_free_all(void)
+{
+  if (transport_list) {
+    clear_transport_list();
+    smartlist_free(transport_list);
+    transport_list = NULL;
+  }
+
+  if (managed_proxy_list) {
+    /* If the proxy is in PT_PROTO_COMPLETED, it has registered its
+       transports and it's the duty of the circuitbuild.c subsystem to
+       free them. Otherwise, it hasn't registered its transports yet
+       and we should free them here. */
+    SMARTLIST_FOREACH(managed_proxy_list, managed_proxy_t *, mp, {
+        SMARTLIST_DEL_CURRENT(managed_proxy_list, mp);
+        managed_proxy_destroy(mp, 1);
+    });
+
+    smartlist_free(managed_proxy_list);
+    managed_proxy_list=NULL;
+  }
+}
+
+/** Return a newly allocated string equal to <b>string</b>, except that every
+ * character in <b>chars_to_escape</b> is preceded by a backslash. */
+char *
+tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape)
+{
+  char *new_string = NULL;
+  char *new_cp = NULL;
+  size_t length, new_length;
+
+  tor_assert(string);
+
+  length = strlen(string);
+
+  if (!length) /* If we were given the empty string, return the same. */
+    return tor_strdup("");
+  /* (new_length > SIZE_MAX) => ((length * 2) + 1 > SIZE_MAX) =>
+     (length*2 > SIZE_MAX - 1) => (length > (SIZE_MAX - 1)/2) */
+  if (length > (SIZE_MAX - 1)/2) /* check for overflow */
+    return NULL;
+
+  /* this should be enough even if all characters must be escaped */
+  new_length = (length * 2) + 1;
+
+  new_string = new_cp = tor_malloc(new_length);
+
+  while (*string) {
+    if (strchr(chars_to_escape, *string))
+      *new_cp++ = '\\';
+
+    *new_cp++ = *string++;
+  }
+
+  *new_cp = '\0'; /* NUL-terminate the new string */
+
+  return new_string;
+}
diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h
new file mode 100644
index 0000000000..d304dcd485
--- /dev/null
+++ b/src/feature/client/transports.h
@@ -0,0 +1,147 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file transports.h
+ * \brief Headers for transports.c
+ **/
+
+#ifndef TOR_TRANSPORTS_H
+#define TOR_TRANSPORTS_H
+
+/** Represents a pluggable transport used by a bridge. */
+typedef struct transport_t {
+  /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */
+  int socks_version;
+  /** Name of pluggable transport protocol */
+  char *name;
+  /** The IP address where the transport bound and is waiting for
+   * connections. */
+  tor_addr_t addr;
+  /** Port of proxy */
+  uint16_t port;
+  /** Boolean: We are re-parsing our transport list, and we are going to remove
+   * this one if we don't find it in the list of configured transports. */
+  unsigned marked_for_removal : 1;
+  /** Arguments for this transport that must be written to the
+      extra-info descriptor. */
+  char *extra_info_args;
+} transport_t;
+
+void mark_transport_list(void);
+void sweep_transport_list(void);
+MOCK_DECL(int, transport_add_from_config,
+          (const tor_addr_t *addr, uint16_t port,
+           const char *name, int socks_ver));
+void transport_free_(transport_t *transport);
+#define transport_free(tr) FREE_AND_NULL(transport_t, transport_free_, (tr))
+
+MOCK_DECL(transport_t*, transport_get_by_name, (const char *name));
+
+MOCK_DECL(void, pt_kickstart_proxy,
+          (const smartlist_t *transport_list, char **proxy_argv,
+           int is_server));
+
+#define pt_kickstart_client_proxy(tl, pa)  \
+  pt_kickstart_proxy(tl, pa, 0)
+#define pt_kickstart_server_proxy(tl, pa) \
+  pt_kickstart_proxy(tl, pa, 1)
+
+void pt_configure_remaining_proxies(void);
+
+int pt_proxies_configuration_pending(void);
+
+char *pt_get_extra_info_descriptor_string(void);
+
+void pt_free_all(void);
+
+void pt_prepare_proxy_list_for_config_read(void);
+void sweep_proxy_list(void);
+
+smartlist_t *get_transport_proxy_ports(void);
+char *pt_stringify_socks_args(const smartlist_t *socks_args);
+
+char *pt_get_socks_args_for_proxy_addrport(const tor_addr_t *addr,
+                                            uint16_t port);
+
+char *tor_escape_str_for_pt_args(const char *string,
+                                 const char *chars_to_escape);
+
+#ifdef PT_PRIVATE
+/** State of the managed proxy configuration protocol. */
+enum pt_proto_state {
+  PT_PROTO_INFANT, /* was just born */
+  PT_PROTO_LAUNCHED, /* was just launched */
+  PT_PROTO_ACCEPTING_METHODS, /* accepting methods */
+  PT_PROTO_CONFIGURED, /* configured successfully */
+  PT_PROTO_COMPLETED, /* configure and registered its transports */
+  PT_PROTO_BROKEN, /* broke during the protocol */
+  PT_PROTO_FAILED_LAUNCH /* failed while launching */
+};
+
+struct process_handle_t;
+
+/** Structure containing information of a managed proxy. */
+typedef struct {
+  enum pt_proto_state conf_state; /* the current configuration state */
+  char **argv; /* the cli arguments of this proxy */
+  int conf_protocol; /* the configuration protocol version used */
+
+  char *proxy_uri;  /* the outgoing proxy in TOR_PT_PROXY URI format */
+  unsigned int proxy_supported : 1; /* the proxy honors TOR_PT_PROXY */
+
+  int is_server; /* is it a server proxy? */
+
+  /* A pointer to the process handle of this managed proxy. */
+  struct process_handle_t *process_handle;
+
+  int pid; /* The Process ID this managed proxy is using. */
+
+  /** Boolean: We are re-parsing our config, and we are going to
+   * remove this managed proxy if we don't find it any transport
+   * plugins that use it. */
+  unsigned int marked_for_removal : 1;
+
+  /** Boolean: We got a SIGHUP while this proxy was running. We use
+   * this flag to signify that this proxy might need to be restarted
+   * so that it can listen for other transports according to the new
+   * torrc. */
+  unsigned int was_around_before_config_read : 1;
+
+  /* transports to-be-launched by this proxy */
+  smartlist_t *transports_to_launch;
+
+  /* The 'transports' list contains all the transports this proxy has
+     launched. */
+  smartlist_t *transports;
+} managed_proxy_t;
+
+STATIC transport_t *transport_new(const tor_addr_t *addr, uint16_t port,
+                                  const char *name, int socks_ver,
+                                  const char *extra_info_args);
+STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp);
+STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp);
+
+STATIC int parse_version(const char *line, managed_proxy_t *mp);
+STATIC void parse_env_error(const char *line);
+STATIC void parse_proxy_error(const char *line);
+STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp);
+STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp);
+
+STATIC void managed_proxy_destroy(managed_proxy_t *mp,
+                                  int also_terminate_process);
+
+STATIC managed_proxy_t *managed_proxy_create(const smartlist_t *transport_list,
+                                             char **proxy_argv, int is_server);
+
+STATIC int configure_proxy(managed_proxy_t *mp);
+
+STATIC char* get_pt_proxy_uri(void);
+
+STATIC void free_execve_args(char **arg);
+
+#endif /* defined(PT_PRIVATE) */
+
+#endif /* !defined(TOR_TRANSPORTS_H) */
diff --git a/src/feature/control/control.c b/src/feature/control/control.c
new file mode 100644
index 0000000000..aa43fccff0
--- /dev/null
+++ b/src/feature/control/control.c
@@ -0,0 +1,7802 @@
+
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control.c
+ * \brief Implementation for Tor's control-socket interface.
+ *
+ * A "controller" is an external program that monitors and controls a Tor
+ * instance via a text-based protocol. It connects to Tor via a connection
+ * to a local socket.
+ *
+ * The protocol is line-driven.  The controller sends commands terminated by a
+ * CRLF.  Tor sends lines that are either <em>replies</em> to what the
+ * controller has said, or <em>events</em> that Tor sends to the controller
+ * asynchronously based on occurrences in the Tor network model.
+ *
+ * See the control-spec.txt file in the torspec.git repository for full
+ * details on protocol.
+ *
+ * This module generally has two kinds of entry points: those based on having
+ * received a command on a controller socket, which are handled in
+ * connection_control_process_inbuf(), and dispatched to individual functions
+ * with names like control_handle_COMMANDNAME(); and those based on events
+ * that occur elsewhere in Tor, which are handled by functions with names like
+ * control_event_EVENTTYPE().
+ *
+ * Controller events are not sent immediately; rather, they are inserted into
+ * the queued_control_events array, and flushed later from
+ * flush_queued_events_cb().  Doing this simplifies our callgraph greatly,
+ * by limiting the number of places in Tor that can call back into the network
+ * stack.
+ **/
+
+#define CONTROL_PRIVATE
+
+#include "or/or.h"
+#include "or/addressmap.h"
+#include "or/bridges.h"
+#include "lib/container/buffers.h"
+#include "or/channel.h"
+#include "or/channeltls.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuitstats.h"
+#include "or/circuituse.h"
+#include "or/command.h"
+#include "lib/evloop/compat_libevent.h"
+#include "or/config.h"
+#include "or/confparse.h"
+#include "or/connection.h"
+#include "or/connection_edge.h"
+#include "or/connection_or.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/dnsserv.h"
+#include "or/entrynodes.h"
+#include "or/geoip.h"
+#include "or/hibernate.h"
+#include "or/hs_cache.h"
+#include "or/hs_common.h"
+#include "or/hs_control.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/proto_control0.h"
+#include "or/proto_http.h"
+#include "or/reasons.h"
+#include "or/rendclient.h"
+#include "or/rendcommon.h"
+#include "or/rendservice.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/shared_random_client.h"
+#include "lib/encoding/confline.h"
+
+#include "or/cached_dir_st.h"
+#include "or/control_connection_st.h"
+#include "or/cpath_build_state_st.h"
+#include "or/entry_connection_st.h"
+#include "or/extrainfo_st.h"
+#include "or/networkstatus_st.h"
+#include "or/node_st.h"
+#include "or/or_connection_st.h"
+#include "or/or_circuit_st.h"
+#include "or/origin_circuit_st.h"
+#include "or/microdesc_st.h"
+#include "or/rend_authorized_client_st.h"
+#include "or/rend_encoded_v2_service_descriptor_st.h"
+#include "or/rend_service_descriptor_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/socks_request_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#ifndef _WIN32
+#include <pwd.h>
+#include <sys/resource.h>
+#endif
+
+#include "lib/crypt_ops/crypto_s2k.h"
+#include "lib/evloop/procmon.h"
+#include "lib/evloop/compat_libevent.h"
+
+/** Yield true iff <b>s</b> is the state of a control_connection_t that has
+ * finished authentication and is accepting commands. */
+#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
+
+/** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
+ * connection is interested in events of type <b>e</b>.  We use this
+ * so that we can decide to skip generating event messages that nobody
+ * has interest in without having to walk over the global connection
+ * list to find out.
+ **/
+typedef uint64_t event_mask_t;
+
+/** An event mask of all the events that any controller is interested in
+ * receiving. */
+static event_mask_t global_event_mask = 0;
+
+/** True iff we have disabled log messages from being sent to the controller */
+static int disable_log_messages = 0;
+
+/** Macro: true if any control connection is interested in events of type
+ * <b>e</b>. */
+#define EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & EVENT_MASK_(e)))
+
+/** Macro: true if any event from the bitfield 'e' is interesting. */
+#define ANY_EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & (e)))
+
+/** If we're using cookie-type authentication, how long should our cookies be?
+ */
+#define AUTHENTICATION_COOKIE_LEN 32
+
+/** If true, we've set authentication_cookie to a secret code and
+ * stored it to disk. */
+static int authentication_cookie_is_set = 0;
+/** If authentication_cookie_is_set, a secret cookie that we've stored to disk
+ * and which we're using to authenticate controllers.  (If the controller can
+ * read it off disk, it has permission to connect.) */
+static uint8_t *authentication_cookie = NULL;
+
+#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \
+  "Tor safe cookie authentication server-to-controller hash"
+#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \
+  "Tor safe cookie authentication controller-to-server hash"
+#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN
+
+/** The list of onion services that have been added via ADD_ONION that do not
+ * belong to any particular control connection.
+ */
+static smartlist_t *detached_onion_services = NULL;
+
+/** A sufficiently large size to record the last bootstrap phase string. */
+#define BOOTSTRAP_MSG_LEN 1024
+
+/** What was the last bootstrap phase message we sent? We keep track
+ * of this so we can respond to getinfo status/bootstrap-phase queries. */
+static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN];
+
+static void connection_printf_to_buf(control_connection_t *conn,
+                                     const char *format, ...)
+  CHECK_PRINTF(2,3);
+static void send_control_event_impl(uint16_t event,
+                                    const char *format, va_list ap)
+  CHECK_PRINTF(2,0);
+static int control_event_status(int type, int severity, const char *format,
+                                va_list args)
+  CHECK_PRINTF(3,0);
+
+static void send_control_done(control_connection_t *conn);
+static void send_control_event(uint16_t event,
+                               const char *format, ...)
+  CHECK_PRINTF(2,3);
+static int handle_control_setconf(control_connection_t *conn, uint32_t len,
+                                  char *body);
+static int handle_control_resetconf(control_connection_t *conn, uint32_t len,
+                                    char *body);
+static int handle_control_getconf(control_connection_t *conn, uint32_t len,
+                                  const char *body);
+static int handle_control_loadconf(control_connection_t *conn, uint32_t len,
+                                  const char *body);
+static int handle_control_setevents(control_connection_t *conn, uint32_t len,
+                                    const char *body);
+static int handle_control_authenticate(control_connection_t *conn,
+                                       uint32_t len,
+                                       const char *body);
+static int handle_control_signal(control_connection_t *conn, uint32_t len,
+                                 const char *body);
+static int handle_control_mapaddress(control_connection_t *conn, uint32_t len,
+                                     const char *body);
+static char *list_getinfo_options(void);
+static int handle_control_getinfo(control_connection_t *conn, uint32_t len,
+                                  const char *body);
+static int handle_control_extendcircuit(control_connection_t *conn,
+                                        uint32_t len,
+                                        const char *body);
+static int handle_control_setcircuitpurpose(control_connection_t *conn,
+                                            uint32_t len, const char *body);
+static int handle_control_attachstream(control_connection_t *conn,
+                                       uint32_t len,
+                                        const char *body);
+static int handle_control_postdescriptor(control_connection_t *conn,
+                                         uint32_t len,
+                                         const char *body);
+static int handle_control_redirectstream(control_connection_t *conn,
+                                         uint32_t len,
+                                         const char *body);
+static int handle_control_closestream(control_connection_t *conn, uint32_t len,
+                                      const char *body);
+static int handle_control_closecircuit(control_connection_t *conn,
+                                       uint32_t len,
+                                       const char *body);
+static int handle_control_resolve(control_connection_t *conn, uint32_t len,
+                                  const char *body);
+static int handle_control_usefeature(control_connection_t *conn,
+                                     uint32_t len,
+                                     const char *body);
+static int handle_control_hsfetch(control_connection_t *conn, uint32_t len,
+                                  const char *body);
+static int handle_control_hspost(control_connection_t *conn, uint32_t len,
+                                 const char *body);
+static int handle_control_add_onion(control_connection_t *conn, uint32_t len,
+                                    const char *body);
+static int handle_control_del_onion(control_connection_t *conn, uint32_t len,
+                                    const char *body);
+static int write_stream_target_to_buf(entry_connection_t *conn, char *buf,
+                                      size_t len);
+static void orconn_target_get_name(char *buf, size_t len,
+                                   or_connection_t *conn);
+
+static int get_cached_network_liveness(void);
+static void set_cached_network_liveness(int liveness);
+
+static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
+
+static char * download_status_to_string(const download_status_t *dl);
+static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
+
+/** Convert a connection_t* to an control_connection_t*; assert if the cast is
+ * invalid. */
+control_connection_t *
+TO_CONTROL_CONN(connection_t *c)
+{
+  tor_assert(c->magic == CONTROL_CONNECTION_MAGIC);
+  return DOWNCAST(control_connection_t, c);
+}
+
+/** Given a control event code for a message event, return the corresponding
+ * log severity. */
+static inline int
+event_to_log_severity(int event)
+{
+  switch (event) {
+    case EVENT_DEBUG_MSG: return LOG_DEBUG;
+    case EVENT_INFO_MSG: return LOG_INFO;
+    case EVENT_NOTICE_MSG: return LOG_NOTICE;
+    case EVENT_WARN_MSG: return LOG_WARN;
+    case EVENT_ERR_MSG: return LOG_ERR;
+    default: return -1;
+  }
+}
+
+/** Given a log severity, return the corresponding control event code. */
+static inline int
+log_severity_to_event(int severity)
+{
+  switch (severity) {
+    case LOG_DEBUG: return EVENT_DEBUG_MSG;
+    case LOG_INFO: return EVENT_INFO_MSG;
+    case LOG_NOTICE: return EVENT_NOTICE_MSG;
+    case LOG_WARN: return EVENT_WARN_MSG;
+    case LOG_ERR: return EVENT_ERR_MSG;
+    default: return -1;
+  }
+}
+
+/** Helper: clear bandwidth counters of all origin circuits. */
+static void
+clear_circ_bw_fields(void)
+{
+  origin_circuit_t *ocirc;
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
+    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+  }
+  SMARTLIST_FOREACH_END(circ);
+}
+
+/** Set <b>global_event_mask*</b> to the bitwise OR of each live control
+ * connection's event_mask field. */
+void
+control_update_global_event_mask(void)
+{
+  smartlist_t *conns = get_connection_array();
+  event_mask_t old_mask, new_mask;
+  old_mask = global_event_mask;
+  int any_old_per_sec_events = control_any_per_second_event_enabled();
+
+  global_event_mask = 0;
+  SMARTLIST_FOREACH(conns, connection_t *, _conn,
+  {
+    if (_conn->type == CONN_TYPE_CONTROL &&
+        STATE_IS_OPEN(_conn->state)) {
+      control_connection_t *conn = TO_CONTROL_CONN(_conn);
+      global_event_mask |= conn->event_mask;
+    }
+  });
+
+  new_mask = global_event_mask;
+
+  /* Handle the aftermath.  Set up the log callback to tell us only what
+   * we want to hear...*/
+  control_adjust_event_log_severity();
+
+  /* Macro: true if ev was false before and is true now. */
+#define NEWLY_ENABLED(ev) \
+  (! (old_mask & (ev)) && (new_mask & (ev)))
+
+  /* ...then, if we've started logging stream or circ bw, clear the
+   * appropriate fields. */
+  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
+    SMARTLIST_FOREACH(conns, connection_t *, conn,
+    {
+      if (conn->type == CONN_TYPE_AP) {
+        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
+        edge_conn->n_written = edge_conn->n_read = 0;
+      }
+    });
+  }
+  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
+    clear_circ_bw_fields();
+  }
+  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
+    uint64_t r, w;
+    control_get_bytes_rw_last_sec(&r, &w);
+  }
+  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
+    reschedule_per_second_timer();
+  }
+
+#undef NEWLY_ENABLED
+}
+
+/** Adjust the log severities that result in control_event_logmsg being called
+ * to match the severity of log messages that any controllers are interested
+ * in. */
+void
+control_adjust_event_log_severity(void)
+{
+  int i;
+  int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG;
+
+  for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      min_log_event = i;
+      break;
+    }
+  }
+  for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      max_log_event = i;
+      break;
+    }
+  }
+  if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) {
+    if (min_log_event > EVENT_NOTICE_MSG)
+      min_log_event = EVENT_NOTICE_MSG;
+    if (max_log_event < EVENT_ERR_MSG)
+      max_log_event = EVENT_ERR_MSG;
+  }
+  if (min_log_event <= max_log_event)
+    change_callback_log_severity(event_to_log_severity(min_log_event),
+                                 event_to_log_severity(max_log_event),
+                                 control_event_logmsg);
+  else
+    change_callback_log_severity(LOG_ERR, LOG_ERR,
+                                 control_event_logmsg);
+}
+
+/** Return true iff the event with code <b>c</b> is being sent to any current
+ * control connection.  This is useful if the amount of work needed to prepare
+ * to call the appropriate control_event_...() function is high.
+ */
+int
+control_event_is_interesting(int event)
+{
+  return EVENT_IS_INTERESTING(event);
+}
+
+/** Return true if any event that needs to fire once a second is enabled. */
+int
+control_any_per_second_event_enabled(void)
+{
+  return ANY_EVENT_IS_INTERESTING(
+      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CELL_STATS) |
+      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CONN_BW) |
+      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
+  );
+}
+
+/* The value of 'get_bytes_read()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_read = 0;
+/* The value of 'get_bytes_written()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_written = 0;
+
+/**
+ * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
+ * and written by Tor since the last call to this function.
+ *
+ * Call this only from the main thread.
+ */
+static void
+control_get_bytes_rw_last_sec(uint64_t *n_read,
+                              uint64_t *n_written)
+{
+  const uint64_t stats_n_bytes_read = get_bytes_read();
+  const uint64_t stats_n_bytes_written = get_bytes_written();
+
+  *n_read = stats_n_bytes_read - stats_prev_n_read;
+  *n_written = stats_n_bytes_written - stats_prev_n_written;
+  stats_prev_n_read = stats_n_bytes_read;
+  stats_prev_n_written = stats_n_bytes_written;
+}
+
+/**
+ * Run all the controller events (if any) that are scheduled to trigger once
+ * per second.
+ */
+void
+control_per_second_events(void)
+{
+  if (!control_any_per_second_event_enabled())
+    return;
+
+  uint64_t bytes_read, bytes_written;
+  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
+  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
+
+  control_event_stream_bandwidth_used();
+  control_event_conn_bandwidth_used();
+  control_event_circ_bandwidth_used();
+  control_event_circuit_cell_stats();
+}
+
+/** Append a NUL-terminated string <b>s</b> to the end of
+ * <b>conn</b>-\>outbuf.
+ */
+static inline void
+connection_write_str_to_buf(const char *s, control_connection_t *conn)
+{
+  size_t len = strlen(s);
+  connection_buf_add(s, len, TO_CONN(conn));
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the
+ * contents of <b>data</b> into *<b>out</b>, adding a period before any period
+ * that appears at the start of a line, and adding a period-CRLF line at
+ * the end. Replace all LF characters sequences with CRLF.  Return the number
+ * of bytes in *<b>out</b>.
+ */
+STATIC size_t
+write_escaped_data(const char *data, size_t len, char **out)
+{
+  tor_assert(len < SIZE_MAX - 9);
+  size_t sz_out = len+8+1;
+  char *outp;
+  const char *start = data, *end;
+  size_t i;
+  int start_of_line;
+  for (i=0; i < len; ++i) {
+    if (data[i] == '\n') {
+      sz_out += 2; /* Maybe add a CR; maybe add a dot. */
+      if (sz_out >= SIZE_T_CEILING) {
+        log_warn(LD_BUG, "Input to write_escaped_data was too long");
+        *out = tor_strdup(".\r\n");
+        return 3;
+      }
+    }
+  }
+  *out = outp = tor_malloc(sz_out);
+  end = data+len;
+  start_of_line = 1;
+  while (data < end) {
+    if (*data == '\n') {
+      if (data > start && data[-1] != '\r')
+        *outp++ = '\r';
+      start_of_line = 1;
+    } else if (*data == '.') {
+      if (start_of_line) {
+        start_of_line = 0;
+        *outp++ = '.';
+      }
+    } else {
+      start_of_line = 0;
+    }
+    *outp++ = *data++;
+  }
+  if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) {
+    *outp++ = '\r';
+    *outp++ = '\n';
+  }
+  *outp++ = '.';
+  *outp++ = '\r';
+  *outp++ = '\n';
+  *outp = '\0'; /* NUL-terminate just in case. */
+  tor_assert(outp >= *out);
+  tor_assert((size_t)(outp - *out) <= sz_out);
+  return outp - *out;
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy
+ * the contents of <b>data</b> into *<b>out</b>, removing any period
+ * that appears at the start of a line, and replacing all CRLF sequences
+ * with LF.   Return the number of
+ * bytes in *<b>out</b>. */
+STATIC size_t
+read_escaped_data(const char *data, size_t len, char **out)
+{
+  char *outp;
+  const char *next;
+  const char *end;
+
+  *out = outp = tor_malloc(len+1);
+
+  end = data+len;
+
+  while (data < end) {
+    /* we're at the start of a line. */
+    if (*data == '.')
+      ++data;
+    next = memchr(data, '\n', end-data);
+    if (next) {
+      size_t n_to_copy = next-data;
+      /* Don't copy a CR that precedes this LF. */
+      if (n_to_copy && *(next-1) == '\r')
+        --n_to_copy;
+      memcpy(outp, data, n_to_copy);
+      outp += n_to_copy;
+      data = next+1; /* This will point at the start of the next line,
+                      * or the end of the string, or a period. */
+    } else {
+      memcpy(outp, data, end-data);
+      outp += (end-data);
+      *outp = '\0';
+      return outp - *out;
+    }
+    *outp++ = '\n';
+  }
+
+  *outp = '\0';
+  return outp - *out;
+}
+
+/** If the first <b>in_len_max</b> characters in <b>start</b> contain a
+ * double-quoted string with escaped characters, return the length of that
+ * string (as encoded, including quotes).  Otherwise return -1. */
+static inline int
+get_escaped_string_length(const char *start, size_t in_len_max,
+                          int *chars_out)
+{
+  const char *cp, *end;
+  int chars = 0;
+
+  if (*start != '\"')
+    return -1;
+
+  cp = start+1;
+  end = start+in_len_max;
+
+  /* Calculate length. */
+  while (1) {
+    if (cp >= end) {
+      return -1; /* Too long. */
+    } else if (*cp == '\\') {
+      if (++cp == end)
+        return -1; /* Can't escape EOS. */
+      ++cp;
+      ++chars;
+    } else if (*cp == '\"') {
+      break;
+    } else {
+      ++cp;
+      ++chars;
+    }
+  }
+  if (chars_out)
+    *chars_out = chars;
+  return (int)(cp - start+1);
+}
+
+/** As decode_escaped_string, but does not decode the string: copies the
+ * entire thing, including quotation marks. */
+static const char *
+extract_escaped_string(const char *start, size_t in_len_max,
+                       char **out, size_t *out_len)
+{
+  int length = get_escaped_string_length(start, in_len_max, NULL);
+  if (length<0)
+    return NULL;
+  *out_len = length;
+  *out = tor_strndup(start, *out_len);
+  return start+length;
+}
+
+/** Given a pointer to a string starting at <b>start</b> containing
+ * <b>in_len_max</b> characters, decode a string beginning with one double
+ * quote, containing any number of non-quote characters or characters escaped
+ * with a backslash, and ending with a final double quote.  Place the resulting
+ * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>;
+ * store its length in <b>out_len</b>.  On success, return a pointer to the
+ * character immediately following the escaped string.  On failure, return
+ * NULL. */
+static const char *
+decode_escaped_string(const char *start, size_t in_len_max,
+                   char **out, size_t *out_len)
+{
+  const char *cp, *end;
+  char *outp;
+  int len, n_chars = 0;
+
+  len = get_escaped_string_length(start, in_len_max, &n_chars);
+  if (len<0)
+    return NULL;
+
+  end = start+len-1; /* Index of last quote. */
+  tor_assert(*end == '\"');
+  outp = *out = tor_malloc(len+1);
+  *out_len = n_chars;
+
+  cp = start+1;
+  while (cp < end) {
+    if (*cp == '\\')
+      ++cp;
+    *outp++ = *cp++;
+  }
+  *outp = '\0';
+  tor_assert((outp - *out) == (int)*out_len);
+
+  return end+1;
+}
+
+/** Create and add a new controller connection on <b>sock</b>.  If
+ * <b>CC_LOCAL_FD_IS_OWNER</b> is set in <b>flags</b>, this Tor process should
+ * exit when the connection closes.  If <b>CC_LOCAL_FD_IS_AUTHENTICATED</b>
+ * is set, then the connection does not need to authenticate.
+ */
+int
+control_connection_add_local_fd(tor_socket_t sock, unsigned flags)
+{
+  if (BUG(! SOCKET_OK(sock)))
+    return -1;
+  const int is_owner = !!(flags & CC_LOCAL_FD_IS_OWNER);
+  const int is_authenticated = !!(flags & CC_LOCAL_FD_IS_AUTHENTICATED);
+  control_connection_t *control_conn = control_connection_new(AF_UNSPEC);
+  connection_t *conn = TO_CONN(control_conn);
+  conn->s = sock;
+  tor_addr_make_unspec(&conn->addr);
+  conn->port = 1;
+  conn->address = tor_strdup("<local socket>");
+
+  /* We take ownership of this socket so that later, when we close it,
+   * we don't freak out. */
+  tor_take_socket_ownership(sock);
+
+  if (set_socket_nonblocking(sock) < 0 ||
+      connection_add(conn) < 0) {
+    connection_free(conn);
+    return -1;
+  }
+
+  control_conn->is_owning_control_connection = is_owner;
+
+  if (connection_init_accepted_conn(conn, NULL) < 0) {
+    connection_mark_for_close(conn);
+    return -1;
+  }
+
+  if (is_authenticated) {
+    conn->state = CONTROL_CONN_STATE_OPEN;
+  }
+
+  return 0;
+}
+
+/** Acts like sprintf, but writes its formatted string to the end of
+ * <b>conn</b>-\>outbuf. */
+static void
+connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
+{
+  va_list ap;
+  char *buf = NULL;
+  int len;
+
+  va_start(ap,format);
+  len = tor_vasprintf(&buf, format, ap);
+  va_end(ap);
+
+  if (len < 0) {
+    log_err(LD_BUG, "Unable to format string for controller.");
+    tor_assert(0);
+  }
+
+  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
+
+  tor_free(buf);
+}
+
+/** Write all of the open control ports to ControlPortWriteToFile */
+void
+control_ports_write_to_file(void)
+{
+  smartlist_t *lines;
+  char *joined = NULL;
+  const or_options_t *options = get_options();
+
+  if (!options->ControlPortWriteToFile)
+    return;
+
+  lines = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(get_connection_array(), const connection_t *, conn) {
+    if (conn->type != CONN_TYPE_CONTROL_LISTENER || conn->marked_for_close)
+      continue;
+#ifdef AF_UNIX
+    if (conn->socket_family == AF_UNIX) {
+      smartlist_add_asprintf(lines, "UNIX_PORT=%s\n", conn->address);
+      continue;
+    }
+#endif /* defined(AF_UNIX) */
+    smartlist_add_asprintf(lines, "PORT=%s:%d\n", conn->address, conn->port);
+  } SMARTLIST_FOREACH_END(conn);
+
+  joined = smartlist_join_strings(lines, "", 0, NULL);
+
+  if (write_str_to_file(options->ControlPortWriteToFile, joined, 0) < 0) {
+    log_warn(LD_CONTROL, "Writing %s failed: %s",
+             options->ControlPortWriteToFile, strerror(errno));
+  }
+#ifndef _WIN32
+  if (options->ControlPortFileGroupReadable) {
+    if (chmod(options->ControlPortWriteToFile, 0640)) {
+      log_warn(LD_FS,"Unable to make %s group-readable.",
+               options->ControlPortWriteToFile);
+    }
+  }
+#endif /* !defined(_WIN32) */
+  tor_free(joined);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+}
+
+/** Send a "DONE" message down the control connection <b>conn</b>. */
+static void
+send_control_done(control_connection_t *conn)
+{
+  connection_write_str_to_buf("250 OK\r\n", conn);
+}
+
+/** Represents an event that's queued to be sent to one or more
+ * controllers. */
+typedef struct queued_event_s {
+  uint16_t event;
+  char *msg;
+} queued_event_t;
+
+/** Pointer to int. If this is greater than 0, we don't allow new events to be
+ * queued. */
+static tor_threadlocal_t block_event_queue_flag;
+
+/** Holds a smartlist of queued_event_t objects that may need to be sent
+ * to one or more controllers */
+static smartlist_t *queued_control_events = NULL;
+
+/** True if the flush_queued_events_event is pending. */
+static int flush_queued_event_pending = 0;
+
+/** Lock to protect the above fields. */
+static tor_mutex_t *queued_control_events_lock = NULL;
+
+/** An event that should fire in order to flush the contents of
+ * queued_control_events. */
+static mainloop_event_t *flush_queued_events_event = NULL;
+
+void
+control_initialize_event_queue(void)
+{
+  if (queued_control_events == NULL) {
+    queued_control_events = smartlist_new();
+  }
+
+  if (flush_queued_events_event == NULL) {
+    struct event_base *b = tor_libevent_get_base();
+    if (b) {
+      flush_queued_events_event =
+        mainloop_event_new(flush_queued_events_cb, NULL);
+      tor_assert(flush_queued_events_event);
+    }
+  }
+
+  if (queued_control_events_lock == NULL) {
+    queued_control_events_lock = tor_mutex_new();
+    tor_threadlocal_init(&block_event_queue_flag);
+  }
+}
+
+static int *
+get_block_event_queue(void)
+{
+  int *val = tor_threadlocal_get(&block_event_queue_flag);
+  if (PREDICT_UNLIKELY(val == NULL)) {
+    val = tor_malloc_zero(sizeof(int));
+    tor_threadlocal_set(&block_event_queue_flag, val);
+  }
+  return val;
+}
+
+/** Helper: inserts an event on the list of events queued to be sent to
+ * one or more controllers, and schedules the events to be flushed if needed.
+ *
+ * This function takes ownership of <b>msg</b>, and may free it.
+ *
+ * We queue these events rather than send them immediately in order to break
+ * the dependency in our callgraph from code that generates events for the
+ * controller, and the network layer at large.  Otherwise, nearly every
+ * interesting part of Tor would potentially call every other interesting part
+ * of Tor.
+ */
+MOCK_IMPL(STATIC void,
+queue_control_event_string,(uint16_t event, char *msg))
+{
+  /* This is redundant with checks done elsewhere, but it's a last-ditch
+   * attempt to avoid queueing something we shouldn't have to queue. */
+  if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) {
+    tor_free(msg);
+    return;
+  }
+
+  int *block_event_queue = get_block_event_queue();
+  if (*block_event_queue) {
+    tor_free(msg);
+    return;
+  }
+
+  queued_event_t *ev = tor_malloc(sizeof(*ev));
+  ev->event = event;
+  ev->msg = msg;
+
+  /* No queueing an event while queueing an event */
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  tor_assert(queued_control_events);
+  smartlist_add(queued_control_events, ev);
+
+  int activate_event = 0;
+  if (! flush_queued_event_pending && in_main_thread()) {
+    activate_event = 1;
+    flush_queued_event_pending = 1;
+  }
+
+  tor_mutex_release(queued_control_events_lock);
+
+  --*block_event_queue;
+
+  /* We just put an event on the queue; mark the queue to be
+   * flushed.  We only do this from the main thread for now; otherwise,
+   * we'd need to incur locking overhead in Libevent or use a socket.
+   */
+  if (activate_event) {
+    tor_assert(flush_queued_events_event);
+    mainloop_event_activate(flush_queued_events_event);
+  }
+}
+
+#define queued_event_free(ev) \
+  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
+
+/** Release all storage held by <b>ev</b>. */
+static void
+queued_event_free_(queued_event_t *ev)
+{
+  if (ev == NULL)
+    return;
+
+  tor_free(ev->msg);
+  tor_free(ev);
+}
+
+/** Send every queued event to every controller that's interested in it,
+ * and remove the events from the queue.  If <b>force</b> is true,
+ * then make all controllers send their data out immediately, since we
+ * may be about to shut down. */
+static void
+queued_events_flush_all(int force)
+{
+  /* Make sure that we get all the pending log events, if there are any. */
+  flush_pending_log_callbacks();
+
+  if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
+    return;
+  }
+  smartlist_t *all_conns = get_connection_array();
+  smartlist_t *controllers = smartlist_new();
+  smartlist_t *queued_events;
+
+  int *block_event_queue = get_block_event_queue();
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  /* No queueing an event while flushing events. */
+  flush_queued_event_pending = 0;
+  queued_events = queued_control_events;
+  queued_control_events = smartlist_new();
+  tor_mutex_release(queued_control_events_lock);
+
+  /* Gather all the controllers that will care... */
+  SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_CONTROL &&
+        !conn->marked_for_close &&
+        conn->state == CONTROL_CONN_STATE_OPEN) {
+      control_connection_t *control_conn = TO_CONTROL_CONN(conn);
+
+      smartlist_add(controllers, control_conn);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) {
+    const event_mask_t bit = ((event_mask_t)1) << ev->event;
+    const size_t msg_len = strlen(ev->msg);
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      if (control_conn->event_mask & bit) {
+        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
+      }
+    } SMARTLIST_FOREACH_END(control_conn);
+
+    queued_event_free(ev);
+  } SMARTLIST_FOREACH_END(ev);
+
+  if (force) {
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      connection_flush(TO_CONN(control_conn));
+    } SMARTLIST_FOREACH_END(control_conn);
+  }
+
+  smartlist_free(queued_events);
+  smartlist_free(controllers);
+
+  --*block_event_queue;
+}
+
+/** Libevent callback: Flushes pending events to controllers that are
+ * interested in them. */
+static void
+flush_queued_events_cb(mainloop_event_t *event, void *arg)
+{
+  (void) event;
+  (void) arg;
+  queued_events_flush_all(0);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is given by <b>msg</b>.
+ *
+ * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with
+ * respect to the EXTENDED_EVENTS feature. */
+MOCK_IMPL(STATIC void,
+send_control_event_string,(uint16_t event,
+                           const char *msg))
+{
+  tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_);
+  queue_control_event_string(event, tor_strdup(msg));
+}
+
+/** Helper for send_control_event and control_event_status:
+ * Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event_impl(uint16_t event,
+                        const char *format, va_list ap)
+{
+  char *buf = NULL;
+  int len;
+
+  len = tor_vasprintf(&buf, format, ap);
+  if (len < 0) {
+    log_warn(LD_BUG, "Unable to format event for controller.");
+    return;
+  }
+
+  queue_control_event_string(event, buf);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event(uint16_t event,
+                   const char *format, ...)
+{
+  va_list ap;
+  va_start(ap, format);
+  send_control_event_impl(event, format, ap);
+  va_end(ap);
+}
+
+/** Given a text circuit <b>id</b>, return the corresponding circuit. */
+static origin_circuit_t *
+get_circ(const char *id)
+{
+  uint32_t n_id;
+  int ok;
+  n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  return circuit_get_by_global_id(n_id);
+}
+
+/** Given a text stream <b>id</b>, return the corresponding AP connection. */
+static entry_connection_t *
+get_stream(const char *id)
+{
+  uint64_t n_id;
+  int ok;
+  connection_t *conn;
+  n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  conn = connection_get_by_global_id(n_id);
+  if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close)
+    return NULL;
+  return TO_ENTRY_CONN(conn);
+}
+
+/** Helper for setconf and resetconf. Acts like setconf, except
+ * it passes <b>use_defaults</b> on to options_trial_assign().  Modifies the
+ * contents of body.
+ */
+static int
+control_setconf_helper(control_connection_t *conn, uint32_t len, char *body,
+                       int use_defaults)
+{
+  setopt_err_t opt_err;
+  config_line_t *lines=NULL;
+  char *start = body;
+  char *errstring = NULL;
+  const unsigned flags =
+    CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0);
+
+  char *config;
+  smartlist_t *entries = smartlist_new();
+
+  /* We have a string, "body", of the format '(key(=val|="val")?)' entries
+   * separated by space.  break it into a list of configuration entries. */
+  while (*body) {
+    char *eq = body;
+    char *key;
+    char *entry;
+    while (!TOR_ISSPACE(*eq) && *eq != '=')
+      ++eq;
+    key = tor_strndup(body, eq-body);
+    body = eq+1;
+    if (*eq == '=') {
+      char *val=NULL;
+      size_t val_len=0;
+      if (*body != '\"') {
+        char *val_start = body;
+        while (!TOR_ISSPACE(*body))
+          body++;
+        val = tor_strndup(val_start, body-val_start);
+        val_len = strlen(val);
+      } else {
+        body = (char*)extract_escaped_string(body, (len - (body-start)),
+                                             &val, &val_len);
+        if (!body) {
+          connection_write_str_to_buf("551 Couldn't parse string\r\n", conn);
+          SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
+          smartlist_free(entries);
+          tor_free(key);
+          return 0;
+        }
+      }
+      tor_asprintf(&entry, "%s %s", key, val);
+      tor_free(key);
+      tor_free(val);
+    } else {
+      entry = key;
+    }
+    smartlist_add(entries, entry);
+    while (TOR_ISSPACE(*body))
+      ++body;
+  }
+
+  smartlist_add_strdup(entries, "");
+  config = smartlist_join_strings(entries, "\n", 0, NULL);
+  SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
+  smartlist_free(entries);
+
+  if (config_get_lines(config, &lines, 0) < 0) {
+    log_warn(LD_CONTROL,"Controller gave us config lines we can't parse.");
+    connection_write_str_to_buf("551 Couldn't parse configuration\r\n",
+                                conn);
+    tor_free(config);
+    return 0;
+  }
+  tor_free(config);
+
+  opt_err = options_trial_assign(lines, flags, &errstring);
+  {
+    const char *msg;
+    switch (opt_err) {
+      case SETOPT_ERR_MISC:
+        msg = "552 Unrecognized option";
+        break;
+      case SETOPT_ERR_PARSE:
+        msg = "513 Unacceptable option value";
+        break;
+      case SETOPT_ERR_TRANSITION:
+        msg = "553 Transition not allowed";
+        break;
+      case SETOPT_ERR_SETTING:
+      default:
+        msg = "553 Unable to set option";
+        break;
+      case SETOPT_OK:
+        config_free_lines(lines);
+        send_control_done(conn);
+        return 0;
+    }
+    log_warn(LD_CONTROL,
+             "Controller gave us config lines that didn't validate: %s",
+             errstring);
+    connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
+    config_free_lines(lines);
+    tor_free(errstring);
+    return 0;
+  }
+}
+
+/** Called when we receive a SETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body.*/
+static int
+handle_control_setconf(control_connection_t *conn, uint32_t len, char *body)
+{
+  return control_setconf_helper(conn, len, body, 0);
+}
+
+/** Called when we receive a RESETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body. */
+static int
+handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body)
+{
+  return control_setconf_helper(conn, len, body, 1);
+}
+
+/** Called when we receive a GETCONF message.  Parse the request, and
+ * reply with a CONFVALUE or an ERROR message */
+static int
+handle_control_getconf(control_connection_t *conn, uint32_t body_len,
+                       const char *body)
+{
+  smartlist_t *questions = smartlist_new();
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  char *msg = NULL;
+  size_t msg_len;
+  const or_options_t *options = get_options();
+  int i, len;
+
+  (void) body_len; /* body is NUL-terminated; so we can ignore len. */
+  smartlist_split_string(questions, body, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    if (!option_is_recognized(q)) {
+      smartlist_add(unrecognized, (char*) q);
+    } else {
+      config_line_t *answer = option_get_assignment(options,q);
+      if (!answer) {
+        const char *name = option_get_canonical_name(q);
+        smartlist_add_asprintf(answers, "250-%s\r\n", name);
+      }
+
+      while (answer) {
+        config_line_t *next;
+        smartlist_add_asprintf(answers, "250-%s=%s\r\n",
+                     answer->key, answer->value);
+
+        next = answer->next;
+        tor_free(answer->key);
+        tor_free(answer->value);
+        tor_free(answer);
+        answer = next;
+      }
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  if ((len = smartlist_len(unrecognized))) {
+    for (i=0; i < len-1; ++i)
+      connection_printf_to_buf(conn,
+                               "552-Unrecognized configuration key \"%s\"\r\n",
+                               (char*)smartlist_get(unrecognized, i));
+    connection_printf_to_buf(conn,
+                             "552 Unrecognized configuration key \"%s\"\r\n",
+                             (char*)smartlist_get(unrecognized, len-1));
+  } else if ((len = smartlist_len(answers))) {
+    char *tmp = smartlist_get(answers, len-1);
+    tor_assert(strlen(tmp)>4);
+    tmp[3] = ' ';
+    msg = smartlist_join_strings(answers, "", 0, &msg_len);
+    connection_buf_add(msg, msg_len, TO_CONN(conn));
+  } else {
+    connection_write_str_to_buf("250 OK\r\n", conn);
+  }
+
+  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
+  smartlist_free(answers);
+  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
+  smartlist_free(questions);
+  smartlist_free(unrecognized);
+
+  tor_free(msg);
+
+  return 0;
+}
+
+/** Called when we get a +LOADCONF message. */
+static int
+handle_control_loadconf(control_connection_t *conn, uint32_t len,
+                         const char *body)
+{
+  setopt_err_t retval;
+  char *errstring = NULL;
+  const char *msg = NULL;
+  (void) len;
+
+  retval = options_init_from_string(NULL, body, CMD_RUN_TOR, NULL, &errstring);
+
+  if (retval != SETOPT_OK)
+    log_warn(LD_CONTROL,
+             "Controller gave us config file that didn't validate: %s",
+             errstring);
+
+  switch (retval) {
+  case SETOPT_ERR_PARSE:
+    msg = "552 Invalid config file";
+    break;
+  case SETOPT_ERR_TRANSITION:
+    msg = "553 Transition not allowed";
+    break;
+  case SETOPT_ERR_SETTING:
+    msg = "553 Unable to set option";
+    break;
+  case SETOPT_ERR_MISC:
+  default:
+    msg = "550 Unable to load config";
+    break;
+  case SETOPT_OK:
+    break;
+  }
+  if (msg) {
+    if (errstring)
+      connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
+    else
+      connection_printf_to_buf(conn, "%s\r\n", msg);
+  } else {
+    send_control_done(conn);
+  }
+  tor_free(errstring);
+  return 0;
+}
+
+/** Helper structure: maps event values to their names. */
+struct control_event_t {
+  uint16_t event_code;
+  const char *event_name;
+};
+/** Table mapping event values to their names.  Used to implement SETEVENTS
+ * and GETINFO events/names, and to keep they in sync. */
+static const struct control_event_t control_event_table[] = {
+  { EVENT_CIRCUIT_STATUS, "CIRC" },
+  { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" },
+  { EVENT_STREAM_STATUS, "STREAM" },
+  { EVENT_OR_CONN_STATUS, "ORCONN" },
+  { EVENT_BANDWIDTH_USED, "BW" },
+  { EVENT_DEBUG_MSG, "DEBUG" },
+  { EVENT_INFO_MSG, "INFO" },
+  { EVENT_NOTICE_MSG, "NOTICE" },
+  { EVENT_WARN_MSG, "WARN" },
+  { EVENT_ERR_MSG, "ERR" },
+  { EVENT_NEW_DESC, "NEWDESC" },
+  { EVENT_ADDRMAP, "ADDRMAP" },
+  { EVENT_DESCCHANGED, "DESCCHANGED" },
+  { EVENT_NS, "NS" },
+  { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
+  { EVENT_STATUS_CLIENT, "STATUS_CLIENT" },
+  { EVENT_STATUS_SERVER, "STATUS_SERVER" },
+  { EVENT_GUARD, "GUARD" },
+  { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" },
+  { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" },
+  { EVENT_NEWCONSENSUS, "NEWCONSENSUS" },
+  { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" },
+  { EVENT_GOT_SIGNAL, "SIGNAL" },
+  { EVENT_CONF_CHANGED, "CONF_CHANGED"},
+  { EVENT_CONN_BW, "CONN_BW" },
+  { EVENT_CELL_STATS, "CELL_STATS" },
+  { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
+  { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
+  { EVENT_HS_DESC, "HS_DESC" },
+  { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" },
+  { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" },
+  { 0, NULL },
+};
+
+/** Called when we get a SETEVENTS message: update conn->event_mask,
+ * and reply with DONE or ERROR. */
+static int
+handle_control_setevents(control_connection_t *conn, uint32_t len,
+                         const char *body)
+{
+  int event_code;
+  event_mask_t event_mask = 0;
+  smartlist_t *events = smartlist_new();
+
+  (void) len;
+
+  smartlist_split_string(events, body, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
+    {
+      if (!strcasecmp(ev, "EXTENDED") ||
+          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
+        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
+                 "supported.", ev);
+        continue;
+      } else {
+        int i;
+        event_code = -1;
+
+        for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+          if (!strcasecmp(ev, control_event_table[i].event_name)) {
+            event_code = control_event_table[i].event_code;
+            break;
+          }
+        }
+
+        if (event_code == -1) {
+          connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n",
+                                   ev);
+          SMARTLIST_FOREACH(events, char *, e, tor_free(e));
+          smartlist_free(events);
+          return 0;
+        }
+      }
+      event_mask |= (((event_mask_t)1) << event_code);
+    }
+  SMARTLIST_FOREACH_END(ev);
+  SMARTLIST_FOREACH(events, char *, e, tor_free(e));
+  smartlist_free(events);
+
+  conn->event_mask = event_mask;
+
+  control_update_global_event_mask();
+  send_control_done(conn);
+  return 0;
+}
+
+/** Decode the hashed, base64'd passwords stored in <b>passwords</b>.
+ * Return a smartlist of acceptable passwords (unterminated strings of
+ * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on
+ * failure.
+ */
+smartlist_t *
+decode_hashed_passwords(config_line_t *passwords)
+{
+  char decoded[64];
+  config_line_t *cl;
+  smartlist_t *sl = smartlist_new();
+
+  tor_assert(passwords);
+
+  for (cl = passwords; cl; cl = cl->next) {
+    const char *hashed = cl->value;
+
+    if (!strcmpstart(hashed, "16:")) {
+      if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))
+                        != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN
+          || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) {
+        goto err;
+      }
+    } else {
+        if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed))
+            != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) {
+          goto err;
+        }
+    }
+    smartlist_add(sl,
+                  tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN));
+  }
+
+  return sl;
+
+ err:
+  SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp));
+  smartlist_free(sl);
+  return NULL;
+}
+
+/** Called when we get an AUTHENTICATE message.  Check whether the
+ * authentication is valid, and if so, update the connection's state to
+ * OPEN.  Reply with DONE or ERROR.
+ */
+static int
+handle_control_authenticate(control_connection_t *conn, uint32_t len,
+                            const char *body)
+{
+  int used_quoted_string = 0;
+  const or_options_t *options = get_options();
+  const char *errstr = "Unknown error";
+  char *password;
+  size_t password_len;
+  const char *cp;
+  int i;
+  int bad_cookie=0, bad_password=0;
+  smartlist_t *sl = NULL;
+
+  if (!len) {
+    password = tor_strdup("");
+    password_len = 0;
+  } else if (TOR_ISXDIGIT(body[0])) {
+    cp = body;
+    while (TOR_ISXDIGIT(*cp))
+      ++cp;
+    i = (int)(cp - body);
+    tor_assert(i>0);
+    password_len = i/2;
+    password = tor_malloc(password_len + 1);
+    if (base16_decode(password, password_len+1, body, i)
+                      != (int) password_len) {
+      connection_write_str_to_buf(
+            "551 Invalid hexadecimal encoding.  Maybe you tried a plain text "
+            "password?  If so, the standard requires that you put it in "
+            "double quotes.\r\n", conn);
+      connection_mark_for_close(TO_CONN(conn));
+      tor_free(password);
+      return 0;
+    }
+  } else {
+    if (!decode_escaped_string(body, len, &password, &password_len)) {
+      connection_write_str_to_buf("551 Invalid quoted string.  You need "
+            "to put the password in double quotes.\r\n", conn);
+      connection_mark_for_close(TO_CONN(conn));
+      return 0;
+    }
+    used_quoted_string = 1;
+  }
+
+  if (conn->safecookie_client_hash != NULL) {
+    /* The controller has chosen safe cookie authentication; the only
+     * acceptable authentication value is the controller-to-server
+     * response. */
+
+    tor_assert(authentication_cookie_is_set);
+
+    if (password_len != DIGEST256_LEN) {
+      log_warn(LD_CONTROL,
+               "Got safe cookie authentication response with wrong length "
+               "(%d)", (int)password_len);
+      errstr = "Wrong length for safe cookie response.";
+      goto err;
+    }
+
+    if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) {
+      log_warn(LD_CONTROL,
+               "Got incorrect safe cookie authentication response");
+      errstr = "Safe cookie response did not match expected value.";
+      goto err;
+    }
+
+    tor_free(conn->safecookie_client_hash);
+    goto ok;
+  }
+
+  if (!options->CookieAuthentication && !options->HashedControlPassword &&
+      !options->HashedControlSessionPassword) {
+    /* if Tor doesn't demand any stronger authentication, then
+     * the controller can get in with anything. */
+    goto ok;
+  }
+
+  if (options->CookieAuthentication) {
+    int also_password = options->HashedControlPassword != NULL ||
+      options->HashedControlSessionPassword != NULL;
+    if (password_len != AUTHENTICATION_COOKIE_LEN) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got authentication cookie with wrong length "
+                 "(%d)", (int)password_len);
+        errstr = "Wrong length on authentication cookie.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else if (tor_memneq(authentication_cookie, password, password_len)) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got mismatched authentication cookie");
+        errstr = "Authentication cookie did not match expected value.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else {
+      goto ok;
+    }
+  }
+
+  if (options->HashedControlPassword ||
+      options->HashedControlSessionPassword) {
+    int bad = 0;
+    smartlist_t *sl_tmp;
+    char received[DIGEST_LEN];
+    int also_cookie = options->CookieAuthentication;
+    sl = smartlist_new();
+    if (options->HashedControlPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (options->HashedControlSessionPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (bad) {
+      if (!also_cookie) {
+        log_warn(LD_BUG,
+                 "Couldn't decode HashedControlPassword: invalid base16");
+        errstr="Couldn't decode HashedControlPassword value in configuration.";
+        goto err;
+      }
+      bad_password = 1;
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+    } else {
+      SMARTLIST_FOREACH(sl, char *, expected,
+      {
+        secret_to_key_rfc2440(received,DIGEST_LEN,
+                              password,password_len,expected);
+        if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN,
+                      received, DIGEST_LEN))
+          goto ok;
+      });
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+
+      if (used_quoted_string)
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration";
+      else
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration. Maybe you tried a plain text password? "
+          "If so, the standard requires that you put it in double quotes.";
+      bad_password = 1;
+      if (!also_cookie)
+        goto err;
+    }
+  }
+
+  /** We only get here if both kinds of authentication failed. */
+  tor_assert(bad_password && bad_cookie);
+  log_warn(LD_CONTROL, "Bad password or authentication cookie on controller.");
+  errstr = "Password did not match HashedControlPassword *or* authentication "
+    "cookie.";
+
+ err:
+  tor_free(password);
+  connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr);
+  connection_mark_for_close(TO_CONN(conn));
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+ ok:
+  log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT
+           ")", conn->base_.s);
+  send_control_done(conn);
+  conn->base_.state = CONTROL_CONN_STATE_OPEN;
+  tor_free(password);
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+}
+
+/** Called when we get a SAVECONF command. Try to flush the current options to
+ * disk, and report success or failure. */
+static int
+handle_control_saveconf(control_connection_t *conn, uint32_t len,
+                        const char *body)
+{
+  (void) len;
+
+  int force = !strcmpstart(body, "FORCE");
+  const or_options_t *options = get_options();
+  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
+    connection_write_str_to_buf(
+      "551 Unable to write configuration to disk.\r\n", conn);
+  } else {
+    send_control_done(conn);
+  }
+  return 0;
+}
+
+struct signal_t {
+  int sig;
+  const char *signal_name;
+};
+
+static const struct signal_t signal_table[] = {
+  { SIGHUP, "RELOAD" },
+  { SIGHUP, "HUP" },
+  { SIGINT, "SHUTDOWN" },
+  { SIGUSR1, "DUMP" },
+  { SIGUSR1, "USR1" },
+  { SIGUSR2, "DEBUG" },
+  { SIGUSR2, "USR2" },
+  { SIGTERM, "HALT" },
+  { SIGTERM, "TERM" },
+  { SIGTERM, "INT" },
+  { SIGNEWNYM, "NEWNYM" },
+  { SIGCLEARDNSCACHE, "CLEARDNSCACHE"},
+  { SIGHEARTBEAT, "HEARTBEAT"},
+  { 0, NULL },
+};
+
+/** Called when we get a SIGNAL command. React to the provided signal, and
+ * report success or failure. (If the signal results in a shutdown, success
+ * may not be reported.) */
+static int
+handle_control_signal(control_connection_t *conn, uint32_t len,
+                      const char *body)
+{
+  int sig = -1;
+  int i;
+  int n = 0;
+  char *s;
+
+  (void) len;
+
+  while (body[n] && ! TOR_ISSPACE(body[n]))
+    ++n;
+  s = tor_strndup(body, n);
+
+  for (i = 0; signal_table[i].signal_name != NULL; ++i) {
+    if (!strcasecmp(s, signal_table[i].signal_name)) {
+      sig = signal_table[i].sig;
+      break;
+    }
+  }
+
+  if (sig < 0)
+    connection_printf_to_buf(conn, "552 Unrecognized signal code \"%s\"\r\n",
+                             s);
+  tor_free(s);
+  if (sig < 0)
+    return 0;
+
+  send_control_done(conn);
+  /* Flush the "done" first if the signal might make us shut down. */
+  if (sig == SIGTERM || sig == SIGINT)
+    connection_flush(TO_CONN(conn));
+
+  activate_signal(sig);
+
+  return 0;
+}
+
+/** Called when we get a TAKEOWNERSHIP command.  Mark this connection
+ * as an owning connection, so that we will exit if the connection
+ * closes. */
+static int
+handle_control_takeownership(control_connection_t *conn, uint32_t len,
+                             const char *body)
+{
+  (void)len;
+  (void)body;
+
+  conn->is_owning_control_connection = 1;
+
+  log_info(LD_CONTROL, "Control connection %d has taken ownership of this "
+           "Tor instance.",
+           (int)(conn->base_.s));
+
+  send_control_done(conn);
+  return 0;
+}
+
+/** Return true iff <b>addr</b> is unusable as a mapaddress target because of
+ * containing funny characters. */
+static int
+address_is_invalid_mapaddress_target(const char *addr)
+{
+  if (!strcmpstart(addr, "*."))
+    return address_is_invalid_destination(addr+2, 1);
+  else
+    return address_is_invalid_destination(addr, 1);
+}
+
+/** Called when we get a MAPADDRESS command; try to bind all listed addresses,
+ * and report success or failure. */
+static int
+handle_control_mapaddress(control_connection_t *conn, uint32_t len,
+                          const char *body)
+{
+  smartlist_t *elts;
+  smartlist_t *lines;
+  smartlist_t *reply;
+  char *r;
+  size_t sz;
+  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
+
+  lines = smartlist_new();
+  elts = smartlist_new();
+  reply = smartlist_new();
+  smartlist_split_string(lines, body, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  SMARTLIST_FOREACH_BEGIN(lines, char *, line) {
+    tor_strlower(line);
+    smartlist_split_string(elts, line, "=", 0, 2);
+    if (smartlist_len(elts) == 2) {
+      const char *from = smartlist_get(elts,0);
+      const char *to = smartlist_get(elts,1);
+      if (address_is_invalid_mapaddress_target(to)) {
+        smartlist_add_asprintf(reply,
+                     "512-syntax error: invalid address '%s'", to);
+        log_warn(LD_CONTROL,
+                 "Skipping invalid argument '%s' in MapAddress msg", to);
+      } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") ||
+                 !strcmp(from, "::")) {
+        const char type =
+          !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME :
+          (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6);
+        const char *address = addressmap_register_virtual_address(
+                                                     type, tor_strdup(to));
+        if (!address) {
+          smartlist_add_asprintf(reply,
+                       "451-resource exhausted: skipping '%s'", line);
+          log_warn(LD_CONTROL,
+                   "Unable to allocate address for '%s' in MapAddress msg",
+                   safe_str_client(line));
+        } else {
+          smartlist_add_asprintf(reply, "250-%s=%s", address, to);
+        }
+      } else {
+        const char *msg;
+        if (addressmap_register_auto(from, to, 1,
+                                     ADDRMAPSRC_CONTROLLER, &msg) < 0) {
+          smartlist_add_asprintf(reply,
+                                 "512-syntax error: invalid address mapping "
+                                 " '%s': %s", line, msg);
+          log_warn(LD_CONTROL,
+                   "Skipping invalid argument '%s' in MapAddress msg: %s",
+                   line, msg);
+        } else {
+          smartlist_add_asprintf(reply, "250-%s", line);
+        }
+      }
+    } else {
+      smartlist_add_asprintf(reply, "512-syntax error: mapping '%s' is "
+                   "not of expected form 'foo=bar'.", line);
+      log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong "
+                           "number of items.",
+                           safe_str_client(line));
+    }
+    SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
+    smartlist_clear(elts);
+  } SMARTLIST_FOREACH_END(line);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+  smartlist_free(elts);
+
+  if (smartlist_len(reply)) {
+    ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
+    r = smartlist_join_strings(reply, "\r\n", 1, &sz);
+    connection_buf_add(r, sz, TO_CONN(conn));
+    tor_free(r);
+  } else {
+    const char *response =
+      "512 syntax error: not enough arguments to mapaddress.\r\n";
+    connection_buf_add(response, strlen(response), TO_CONN(conn));
+  }
+
+  SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
+  smartlist_free(reply);
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows the answers for various
+ * trivial-to-implement questions. */
+static int
+getinfo_helper_misc(control_connection_t *conn, const char *question,
+                    char **answer, const char **errmsg)
+{
+  (void) conn;
+  if (!strcmp(question, "version")) {
+    *answer = tor_strdup(get_version());
+  } else if (!strcmp(question, "bw-event-cache")) {
+    *answer = get_bw_samples();
+  } else if (!strcmp(question, "config-file")) {
+    const char *a = get_torrc_fname(0);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-defaults-file")) {
+    const char *a = get_torrc_fname(1);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-text")) {
+    *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
+  } else if (!strcmp(question, "config-can-saveconf")) {
+    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
+  } else if (!strcmp(question, "info/names")) {
+    *answer = list_getinfo_options();
+  } else if (!strcmp(question, "dormant")) {
+    int dormant = rep_hist_circbuilding_dormant(time(NULL));
+    *answer = tor_strdup(dormant ? "1" : "0");
+  } else if (!strcmp(question, "events/names")) {
+    int i;
+    smartlist_t *event_names = smartlist_new();
+
+    for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+      smartlist_add(event_names, (char *)control_event_table[i].event_name);
+    }
+
+    *answer = smartlist_join_strings(event_names, " ", 0, NULL);
+
+    smartlist_free(event_names);
+  } else if (!strcmp(question, "signal/names")) {
+    smartlist_t *signal_names = smartlist_new();
+    int j;
+    for (j = 0; signal_table[j].signal_name != NULL; ++j) {
+      smartlist_add(signal_names, (char*)signal_table[j].signal_name);
+    }
+
+    *answer = smartlist_join_strings(signal_names, " ", 0, NULL);
+
+    smartlist_free(signal_names);
+  } else if (!strcmp(question, "features/names")) {
+    *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS");
+  } else if (!strcmp(question, "address")) {
+    uint32_t addr;
+    if (router_pick_published_address(get_options(), &addr, 0) < 0) {
+      *errmsg = "Address unknown";
+      return -1;
+    }
+    *answer = tor_dup_ip(addr);
+  } else if (!strcmp(question, "traffic/read")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
+  } else if (!strcmp(question, "traffic/written")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
+  } else if (!strcmp(question, "process/pid")) {
+    int myPid = -1;
+
+#ifdef _WIN32
+      myPid = _getpid();
+#else
+      myPid = getpid();
+#endif
+
+    tor_asprintf(answer, "%d", myPid);
+  } else if (!strcmp(question, "process/uid")) {
+#ifdef _WIN32
+      *answer = tor_strdup("-1");
+#else
+      int myUid = geteuid();
+      tor_asprintf(answer, "%d", myUid);
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/user")) {
+#ifdef _WIN32
+      *answer = tor_strdup("");
+#else
+      int myUid = geteuid();
+      const struct passwd *myPwEntry = tor_getpwuid(myUid);
+
+      if (myPwEntry) {
+        *answer = tor_strdup(myPwEntry->pw_name);
+      } else {
+        *answer = tor_strdup("");
+      }
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/descriptor-limit")) {
+    int max_fds = get_max_sockets();
+    tor_asprintf(answer, "%d", max_fds);
+  } else if (!strcmp(question, "limits/max-mem-in-queues")) {
+    tor_asprintf(answer, "%"PRIu64,
+                 (get_options()->MaxMemInQueues));
+  } else if (!strcmp(question, "fingerprint")) {
+    crypto_pk_t *server_key;
+    if (!server_mode(get_options())) {
+      *errmsg = "Not running in server mode";
+      return -1;
+    }
+    server_key = get_server_identity_key();
+    *answer = tor_malloc(HEX_DIGEST_LEN+1);
+    crypto_pk_get_fingerprint(server_key, *answer, 0);
+  }
+  return 0;
+}
+
+/** Awful hack: return a newly allocated string based on a routerinfo and
+ * (possibly) an extrainfo, sticking the read-history and write-history from
+ * <b>ei</b> into the resulting string.  The thing you get back won't
+ * necessarily have a valid signature.
+ *
+ * New code should never use this; it's for backward compatibility.
+ *
+ * NOTE: <b>ri_body</b> is as returned by signed_descriptor_get_body: it might
+ * not be NUL-terminated. */
+static char *
+munge_extrainfo_into_routerinfo(const char *ri_body,
+                                const signed_descriptor_t *ri,
+                                const signed_descriptor_t *ei)
+{
+  char *out = NULL, *outp;
+  int i;
+  const char *router_sig;
+  const char *ei_body = signed_descriptor_get_body(ei);
+  size_t ri_len = ri->signed_descriptor_len;
+  size_t ei_len = ei->signed_descriptor_len;
+  if (!ei_body)
+    goto bail;
+
+  outp = out = tor_malloc(ri_len+ei_len+1);
+  if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature")))
+    goto bail;
+  ++router_sig;
+  memcpy(out, ri_body, router_sig-ri_body);
+  outp += router_sig-ri_body;
+
+  for (i=0; i < 2; ++i) {
+    const char *kwd = i ? "\nwrite-history " : "\nread-history ";
+    const char *cp, *eol;
+    if (!(cp = tor_memstr(ei_body, ei_len, kwd)))
+      continue;
+    ++cp;
+    if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body))))
+      continue;
+    memcpy(outp, cp, eol-cp+1);
+    outp += eol-cp+1;
+  }
+  memcpy(outp, router_sig, ri_len - (router_sig-ri_body));
+  *outp++ = '\0';
+  tor_assert(outp-out < (int)(ri_len+ei_len+1));
+
+  return out;
+ bail:
+  tor_free(out);
+  return tor_strndup(ri_body, ri->signed_descriptor_len);
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * which ports are bound. */
+static int
+getinfo_helper_listeners(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  int type;
+  smartlist_t *res;
+
+  (void)control_conn;
+  (void)errmsg;
+
+  if (!strcmp(question, "net/listeners/or"))
+    type = CONN_TYPE_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/dir"))
+    type = CONN_TYPE_DIR_LISTENER;
+  else if (!strcmp(question, "net/listeners/socks"))
+    type = CONN_TYPE_AP_LISTENER;
+  else if (!strcmp(question, "net/listeners/trans"))
+    type = CONN_TYPE_AP_TRANS_LISTENER;
+  else if (!strcmp(question, "net/listeners/natd"))
+    type = CONN_TYPE_AP_NATD_LISTENER;
+  else if (!strcmp(question, "net/listeners/dns"))
+    type = CONN_TYPE_AP_DNS_LISTENER;
+  else if (!strcmp(question, "net/listeners/control"))
+    type = CONN_TYPE_CONTROL_LISTENER;
+  else
+    return 0; /* unknown key */
+
+  res = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
+    struct sockaddr_storage ss;
+    socklen_t ss_len = sizeof(ss);
+
+    if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s))
+      continue;
+
+    if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) {
+      smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port);
+    } else {
+      char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss);
+      smartlist_add(res, esc_for_log(tmp));
+      tor_free(tmp);
+    }
+
+  } SMARTLIST_FOREACH_END(conn);
+
+  *answer = smartlist_join_strings(res, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(res, char *, cp, tor_free(cp));
+  smartlist_free(res);
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * the current time in both local and UTF forms. */
+STATIC int
+getinfo_helper_current_time(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+
+  struct timeval now;
+  tor_gettimeofday(&now);
+  char timebuf[ISO_TIME_LEN+1];
+
+  if (!strcmp(question, "current-time/local"))
+    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else if (!strcmp(question, "current-time/utc"))
+    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else
+    return 0;
+
+  *answer = tor_strdup(timebuf);
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * directory information. */
+STATIC int
+getinfo_helper_dir(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  (void) control_conn;
+  if (!strcmpstart(question, "desc/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/id/* "
+                "instead of desc/id/*.";
+      return 0;
+    }
+  } else if (!strcmpstart(question, "desc/name/")) {
+    const routerinfo_t *ri = NULL;
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node =
+      node_get_by_nickname(question+strlen("desc/name/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/name/* "
+                "instead of desc/name/*.";
+      return 0;
+    }
+  } else if (!strcmp(question, "desc/download-enabled")) {
+    int r = we_fetch_router_descriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmp(question, "desc/all-recent")) {
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri,
+      {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        if (body)
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+      });
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) {
+    /* XXXX Remove this once Torstat asks for extrainfos. */
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest(
+                                     ri->cache_info.extra_info_digest);
+        if (ei && body) {
+          smartlist_add(sl, munge_extrainfo_into_routerinfo(body,
+                                                        &ri->cache_info, ei));
+        } else if (body) {
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+        }
+      } SMARTLIST_FOREACH_END(ri);
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmpstart(question, "hs/client/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/client/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      rend_cache_entry_t *e = NULL;
+      if (!rend_cache_lookup_entry(question, -1, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      const char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_cache_lookup_encoded_as_client(&service_pk);
+      if (desc) {
+        *answer = tor_strdup(desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/service/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+    rend_cache_entry_t *e = NULL;
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_service_lookup_current_desc(&service_pk);
+      if (desc) {
+        /* Newly allocated string, we have ownership. */
+        *answer = desc;
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmp(question, "md/all")) {
+    const smartlist_t *nodes = nodelist_get_list();
+    tor_assert(nodes);
+
+    if (smartlist_len(nodes) == 0) {
+      *answer = tor_strdup("");
+      return 0;
+    }
+
+    smartlist_t *microdescs = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
+      if (n->md && n->md->body) {
+        char *copy = tor_strndup(n->md->body, n->md->bodylen);
+        smartlist_add(microdescs, copy);
+      }
+    } SMARTLIST_FOREACH_END(n);
+
+    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
+    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
+    smartlist_free(microdescs);
+  } else if (!strcmpstart(question, "md/id/")) {
+    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmpstart(question, "md/name/")) {
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
+    /* XXXX duplicated code */
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmp(question, "md/download-enabled")) {
+    int r = we_fetch_microdescriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmpstart(question, "desc-annotations/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node =
+      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *annotations =
+        signed_descriptor_get_annotations(&ri->cache_info);
+      if (annotations)
+        *answer = tor_strndup(annotations,
+                              ri->cache_info.annotations_len);
+    }
+  } else if (!strcmpstart(question, "dir/server/")) {
+    size_t answer_len = 0;
+    char *url = NULL;
+    smartlist_t *descs = smartlist_new();
+    const char *msg;
+    int res;
+    char *cp;
+    tor_asprintf(&url, "/tor/%s", question+4);
+    res = dirserv_get_routerdescs(descs, url, &msg);
+    if (res) {
+      log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg);
+      smartlist_free(descs);
+      tor_free(url);
+      *errmsg = msg;
+      return -1;
+    }
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      answer_len += sd->signed_descriptor_len);
+    cp = *answer = tor_malloc(answer_len+1);
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      {
+                        memcpy(cp, signed_descriptor_get_body(sd),
+                               sd->signed_descriptor_len);
+                        cp += sd->signed_descriptor_len;
+                      });
+    *cp = '\0';
+    tor_free(url);
+    smartlist_free(descs);
+  } else if (!strcmpstart(question, "dir/status/")) {
+    *answer = tor_strdup("");
+  } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
+    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
+      const cached_dir_t *consensus = dirserv_get_consensus("ns");
+      if (consensus)
+        *answer = tor_strdup(consensus->dir);
+    }
+    if (!*answer) { /* try loading it from disk */
+      char *filename = get_cachedir_fname("cached-consensus");
+      *answer = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+      tor_free(filename);
+      if (!*answer) { /* generate an error */
+        *errmsg = "Could not open cached consensus. "
+          "Make sure FetchUselessDescriptors is set to 1.";
+        return -1;
+      }
+    }
+  } else if (!strcmp(question, "network-status")) { /* v1 */
+    static int network_status_warned = 0;
+    if (!network_status_warned) {
+      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
+               "go away in a future version of Tor.");
+      network_status_warned = 1;
+    }
+    routerlist_t *routerlist = router_get_routerlist();
+    if (!routerlist || !routerlist->routers ||
+        list_server_status_v1(routerlist->routers, answer, 1) < 0) {
+      return -1;
+    }
+  } else if (!strcmpstart(question, "extra-info/digest/")) {
+    question += strlen("extra-info/digest/");
+    if (strlen(question) == HEX_DIGEST_LEN) {
+      char d[DIGEST_LEN];
+      signed_descriptor_t *sd = NULL;
+      if (base16_decode(d, sizeof(d), question, strlen(question))
+                        == sizeof(d)) {
+        /* XXXX this test should move into extrainfo_get_by_descriptor_digest,
+         * but I don't want to risk affecting other parts of the code,
+         * especially since the rules for using our own extrainfo (including
+         * when it might be freed) are different from those for using one
+         * we have downloaded. */
+        if (router_extrainfo_digest_is_me(d))
+          sd = &(router_get_my_extrainfo()->cache_info);
+        else
+          sd = extrainfo_get_by_descriptor_digest(d);
+      }
+      if (sd) {
+        const char *body = signed_descriptor_get_body(sd);
+        if (body)
+          *answer = tor_strndup(body, sd->signed_descriptor_len);
+      }
+    }
+  }
+
+  return 0;
+}
+
+/** Given a smartlist of 20-byte digests, return a newly allocated string
+ * containing each of those digests in order, formatted in HEX, and terminated
+ * with a newline. */
+static char *
+digest_list_to_string(const smartlist_t *sl)
+{
+  int len;
+  char *result, *s;
+
+  /* Allow for newlines, and a \0 at the end */
+  len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1;
+  result = tor_malloc_zero(len);
+
+  s = result;
+  SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) {
+    base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN);
+    s[HEX_DIGEST_LEN] = '\n';
+    s += HEX_DIGEST_LEN + 1;
+  } SMARTLIST_FOREACH_END(digest);
+  *s = '\0';
+
+  return result;
+}
+
+/** Turn a download_status_t into a human-readable description in a newly
+ * allocated string.  The format is specified in control-spec.txt, under
+ * the documentation for "GETINFO download/..." .  */
+static char *
+download_status_to_string(const download_status_t *dl)
+{
+  char *rv = NULL;
+  char tbuf[ISO_TIME_LEN+1];
+  const char *schedule_str, *want_authority_str;
+  const char *increment_on_str, *backoff_str;
+
+  if (dl) {
+    /* Get some substrings of the eventual output ready */
+    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
+
+    switch (dl->schedule) {
+      case DL_SCHED_GENERIC:
+        schedule_str = "DL_SCHED_GENERIC";
+        break;
+      case DL_SCHED_CONSENSUS:
+        schedule_str = "DL_SCHED_CONSENSUS";
+        break;
+      case DL_SCHED_BRIDGE:
+        schedule_str = "DL_SCHED_BRIDGE";
+        break;
+      default:
+        schedule_str = "unknown";
+        break;
+    }
+
+    switch (dl->want_authority) {
+      case DL_WANT_ANY_DIRSERVER:
+        want_authority_str = "DL_WANT_ANY_DIRSERVER";
+        break;
+      case DL_WANT_AUTHORITY:
+        want_authority_str = "DL_WANT_AUTHORITY";
+        break;
+      default:
+        want_authority_str = "unknown";
+        break;
+    }
+
+    switch (dl->increment_on) {
+      case DL_SCHED_INCREMENT_FAILURE:
+        increment_on_str = "DL_SCHED_INCREMENT_FAILURE";
+        break;
+      case DL_SCHED_INCREMENT_ATTEMPT:
+        increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT";
+        break;
+      default:
+        increment_on_str = "unknown";
+        break;
+    }
+
+    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
+
+    /* Now assemble them */
+    tor_asprintf(&rv,
+                 "next-attempt-at %s\n"
+                 "n-download-failures %u\n"
+                 "n-download-attempts %u\n"
+                 "schedule %s\n"
+                 "want-authority %s\n"
+                 "increment-on %s\n"
+                 "backoff %s\n"
+                 "last-backoff-position %u\n"
+                 "last-delay-used %d\n",
+                 tbuf,
+                 dl->n_download_failures,
+                 dl->n_download_attempts,
+                 schedule_str,
+                 want_authority_str,
+                 increment_on_str,
+                 backoff_str,
+                 dl->last_backoff_position,
+                 dl->last_delay_used);
+  }
+
+  return rv;
+}
+
+/** Handle the consensus download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_networkstatus(const char *flavor,
+                                       download_status_t **dl_to_emit,
+                                       const char **errmsg)
+{
+  /*
+   * We get the one for the current bootstrapped status by default, or
+   * take an extra /bootstrap or /running suffix
+   */
+  if (strcmp(flavor, "ns") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS);
+  } else if (strcmp(flavor, "ns/bootstrap") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS);
+  } else if (strcmp(flavor, "ns/running") == 0 ) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS);
+  } else if (strcmp(flavor, "microdesc") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/bootstrap") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/running") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC);
+  } else {
+    *errmsg = "Unknown flavor";
+  }
+}
+
+/** Handle the cert download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_cert(const char *fp_sk_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  const char *sk_req;
+  char id_digest[DIGEST_LEN];
+  char sk_digest[DIGEST_LEN];
+
+  /*
+   * We have to handle four cases; fp_sk_req is the request with
+   * a prefix of "downloads/cert/" snipped off.
+   *
+   * Case 1: fp_sk_req = "fps"
+   *  - We should emit a digest_list with a list of all the identity
+   *    fingerprints that can be queried for certificate download status;
+   *    get it by calling list_authority_ids_with_downloads().
+   *
+   * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp
+   *  - We want the default certificate for this identity fingerprint's
+   *    download status; this is the download we get from URLs starting
+   *    in /fp/ on the directory server.  We can get it with
+   *    id_only_download_status_for_authority_id().
+   *
+   * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp
+   *  - We want a list of all signing key digests for this identity
+   *    fingerprint which can be queried for certificate download status.
+   *    Get it with list_sk_digests_for_authority_id().
+   *
+   * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and
+   *         signing key digest sk
+   *   - We want the download status for the certificate for this specific
+   *     signing key and fingerprint.  These correspond to the ones we get
+   *     from URLs starting in /fp-sk/ on the directory server.  Get it with
+   *     list_sk_digests_for_authority_id().
+   */
+
+  if (strcmp(fp_sk_req, "fps") == 0) {
+    *digest_list = list_authority_ids_with_downloads();
+    if (!(*digest_list)) {
+      *errmsg = "Failed to get list of authority identity digests (!)";
+    }
+  } else if (!strcmpstart(fp_sk_req, "fp/")) {
+    fp_sk_req += strlen("fp/");
+    /* Okay, look for another / to tell the fp from fp-sk cases */
+    sk_req = strchr(fp_sk_req, '/');
+    if (sk_req) {
+      /* okay, split it here and try to parse <fp> */
+      if (base16_decode(id_digest, DIGEST_LEN,
+                        fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) {
+        /* Skip past the '/' */
+        ++sk_req;
+        if (strcmp(sk_req, "sks") == 0) {
+          /* We're asking for the list of signing key fingerprints */
+          *digest_list = list_sk_digests_for_authority_id(id_digest);
+          if (!(*digest_list)) {
+            *errmsg = "Failed to get list of signing key digests for this "
+                      "authority identity digest";
+          }
+        } else {
+          /* We've got a signing key digest */
+          if (base16_decode(sk_digest, DIGEST_LEN,
+                            sk_req, strlen(sk_req)) == DIGEST_LEN) {
+            *dl_to_emit =
+              download_status_for_authority_id_and_sk(id_digest, sk_digest);
+            if (!(*dl_to_emit)) {
+              *errmsg = "Failed to get download status for this identity/"
+                        "signing key digest pair";
+            }
+          } else {
+            *errmsg = "That didn't look like a signing key digest";
+          }
+        }
+      } else {
+        *errmsg = "That didn't look like an identity digest";
+      }
+    } else {
+      /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */
+      if (strlen(fp_sk_req) == HEX_DIGEST_LEN) {
+        if (base16_decode(id_digest, DIGEST_LEN,
+                          fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) {
+          *dl_to_emit = id_only_download_status_for_authority_id(id_digest);
+          if (!(*dl_to_emit)) {
+            *errmsg = "Failed to get download status for this authority "
+                      "identity digest";
+          }
+        } else {
+          *errmsg = "That didn't look like a digest";
+        }
+      } else {
+        *errmsg = "That didn't look like a digest";
+      }
+    }
+  } else {
+    *errmsg = "Unknown certificate download status query";
+  }
+}
+
+/** Handle the routerdesc download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_desc(const char *desc_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  char desc_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: desc_req = "descs"
+   *   - Emit a list of all router descriptor digests, which we get by
+   *     calling router_get_descriptor_digests(); this can return NULL
+   *     if we have no current ns-flavor consensus.
+   *
+   * Case 2: desc_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using router_get_dl_status_by_descriptor_digest().
+   */
+
+  if (strcmp(desc_req, "descs") == 0) {
+    *digest_list = router_get_descriptor_digests();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to have a networkstatus-flavored consensus";
+    }
+    /*
+     * Microdescs don't use the download_status_t mechanism, so we don't
+     * answer queries about their downloads here; see microdesc.c.
+     */
+  } else if (strlen(desc_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(desc_digest, DIGEST_LEN,
+                      desc_req, strlen(desc_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such descriptor digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown router descriptor download status query";
+  }
+}
+
+/** Handle the bridge download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_bridge(const char *bridge_req,
+                                download_status_t **dl_to_emit,
+                                smartlist_t **digest_list,
+                                const char **errmsg)
+{
+  char bridge_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: bridge_req = "bridges"
+   *   - Emit a list of all bridge identity digests, which we get by
+   *     calling list_bridge_identities(); this can return NULL if we are
+   *     not using bridges.
+   *
+   * Case 2: bridge_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using get_bridge_dl_status_by_id().
+   */
+
+  if (strcmp(bridge_req, "bridges") == 0) {
+    *digest_list = list_bridge_identities();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to be using bridges";
+    }
+  } else if (strlen(bridge_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(bridge_digest, DIGEST_LEN,
+                      bridge_req, strlen(bridge_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such bridge identity digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown bridge descriptor download status query";
+  }
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * download status information. */
+STATIC int
+getinfo_helper_downloads(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  download_status_t *dl_to_emit = NULL;
+  smartlist_t *digest_list = NULL;
+
+  /* Assert args are sane */
+  tor_assert(control_conn != NULL);
+  tor_assert(question != NULL);
+  tor_assert(answer != NULL);
+  tor_assert(errmsg != NULL);
+
+  /* We check for this later to see if we should supply a default */
+  *errmsg = NULL;
+
+  /* Are we after networkstatus downloads? */
+  if (!strcmpstart(question, "downloads/networkstatus/")) {
+    getinfo_helper_downloads_networkstatus(
+        question + strlen("downloads/networkstatus/"),
+        &dl_to_emit, errmsg);
+  /* Certificates? */
+  } else if (!strcmpstart(question, "downloads/cert/")) {
+    getinfo_helper_downloads_cert(
+        question + strlen("downloads/cert/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Router descriptors? */
+  } else if (!strcmpstart(question, "downloads/desc/")) {
+    getinfo_helper_downloads_desc(
+        question + strlen("downloads/desc/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Bridge descriptors? */
+  } else if (!strcmpstart(question, "downloads/bridge/")) {
+    getinfo_helper_downloads_bridge(
+        question + strlen("downloads/bridge/"),
+        &dl_to_emit, &digest_list, errmsg);
+  } else {
+    *errmsg = "Unknown download status query";
+  }
+
+  if (dl_to_emit) {
+    *answer = download_status_to_string(dl_to_emit);
+
+    return 0;
+  } else if (digest_list) {
+    *answer = digest_list_to_string(digest_list);
+    SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s));
+    smartlist_free(digest_list);
+
+    return 0;
+  } else {
+    if (!(*errmsg)) {
+      *errmsg = "Unknown error";
+    }
+
+    return -1;
+  }
+}
+
+/** Allocate and return a description of <b>circ</b>'s current status,
+ * including its path (if any). */
+static char *
+circuit_describe_status_for_controller(origin_circuit_t *circ)
+{
+  char *rv;
+  smartlist_t *descparts = smartlist_new();
+
+  {
+    char *vpath = circuit_list_path_for_controller(circ);
+    if (*vpath) {
+      smartlist_add(descparts, vpath);
+    } else {
+      tor_free(vpath); /* empty path; don't put an extra space in the result */
+    }
+  }
+
+  {
+    cpath_build_state_t *build_state = circ->build_state;
+    smartlist_t *flaglist = smartlist_new();
+    char *flaglist_joined;
+
+    if (build_state->onehop_tunnel)
+      smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL");
+    if (build_state->is_internal)
+      smartlist_add(flaglist, (void *)"IS_INTERNAL");
+    if (build_state->need_capacity)
+      smartlist_add(flaglist, (void *)"NEED_CAPACITY");
+    if (build_state->need_uptime)
+      smartlist_add(flaglist, (void *)"NEED_UPTIME");
+
+    /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */
+    if (smartlist_len(flaglist)) {
+      flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL);
+
+      smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined);
+
+      tor_free(flaglist_joined);
+    }
+
+    smartlist_free(flaglist);
+  }
+
+  smartlist_add_asprintf(descparts, "PURPOSE=%s",
+                    circuit_purpose_to_controller_string(circ->base_.purpose));
+
+  {
+    const char *hs_state =
+      circuit_purpose_to_controller_hs_state_string(circ->base_.purpose);
+
+    if (hs_state != NULL) {
+      smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state);
+    }
+  }
+
+  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
+    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    const char *onion_address;
+    if (circ->rend_data) {
+      onion_address = rend_data_get_address(circ->rend_data);
+    } else {
+      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
+      onion_address = addr;
+    }
+    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
+  }
+
+  {
+    char tbuf[ISO_TIME_USEC_LEN+1];
+    format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created);
+
+    smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf);
+  }
+
+  // Show username and/or password if available.
+  if (circ->socks_username_len > 0) {
+    char* socks_username_escaped = esc_for_log_len(circ->socks_username,
+                                     (size_t) circ->socks_username_len);
+    smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
+                           socks_username_escaped);
+    tor_free(socks_username_escaped);
+  }
+  if (circ->socks_password_len > 0) {
+    char* socks_password_escaped = esc_for_log_len(circ->socks_password,
+                                     (size_t) circ->socks_password_len);
+    smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
+                           socks_password_escaped);
+    tor_free(socks_password_escaped);
+  }
+
+  rv = smartlist_join_strings(descparts, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
+  smartlist_free(descparts);
+
+  return rv;
+}
+
+/** Implementation helper for GETINFO: knows how to generate summaries of the
+ * current states of things we send events about. */
+static int
+getinfo_helper_events(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  const or_options_t *options = get_options();
+  (void) control_conn;
+  if (!strcmp(question, "circuit-status")) {
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
+      origin_circuit_t *circ;
+      char *circdesc;
+      const char *state;
+      if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close)
+        continue;
+      circ = TO_ORIGIN_CIRCUIT(circ_);
+
+      if (circ->base_.state == CIRCUIT_STATE_OPEN)
+        state = "BUILT";
+      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
+        state = "GUARD_WAIT";
+      else if (circ->cpath)
+        state = "EXTENDED";
+      else
+        state = "LAUNCHED";
+
+      circdesc = circuit_describe_status_for_controller(circ);
+
+      smartlist_add_asprintf(status, "%lu %s%s%s",
+                   (unsigned long)circ->global_identifier,
+                   state, *circdesc ? " " : "", circdesc);
+      tor_free(circdesc);
+    }
+    SMARTLIST_FOREACH_END(circ_);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "stream-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    char buf[256];
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      entry_connection_t *conn;
+      circuit_t *circ;
+      origin_circuit_t *origin_circ = NULL;
+      if (base_conn->type != CONN_TYPE_AP ||
+          base_conn->marked_for_close ||
+          base_conn->state == AP_CONN_STATE_SOCKS_WAIT ||
+          base_conn->state == AP_CONN_STATE_NATD_WAIT)
+        continue;
+      conn = TO_ENTRY_CONN(base_conn);
+      switch (base_conn->state)
+        {
+        case AP_CONN_STATE_CONTROLLER_WAIT:
+        case AP_CONN_STATE_CIRCUIT_WAIT:
+          if (conn->socks_request &&
+              SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command))
+            state = "NEWRESOLVE";
+          else
+            state = "NEW";
+          break;
+        case AP_CONN_STATE_RENDDESC_WAIT:
+        case AP_CONN_STATE_CONNECT_WAIT:
+          state = "SENTCONNECT"; break;
+        case AP_CONN_STATE_RESOLVE_WAIT:
+          state = "SENTRESOLVE"; break;
+        case AP_CONN_STATE_OPEN:
+          state = "SUCCEEDED"; break;
+        default:
+          log_warn(LD_BUG, "Asked for stream in unknown state %d",
+                   base_conn->state);
+          continue;
+        }
+      circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+      if (circ && CIRCUIT_IS_ORIGIN(circ))
+        origin_circ = TO_ORIGIN_CIRCUIT(circ);
+      write_stream_target_to_buf(conn, buf, sizeof(buf));
+      smartlist_add_asprintf(status, "%lu %s %lu %s",
+                   (unsigned long) base_conn->global_identifier,state,
+                   origin_circ?
+                         (unsigned long)origin_circ->global_identifier : 0ul,
+                   buf);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "orconn-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      char name[128];
+      or_connection_t *conn;
+      if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close)
+        continue;
+      conn = TO_OR_CONN(base_conn);
+      if (conn->base_.state == OR_CONN_STATE_OPEN)
+        state = "CONNECTED";
+      else if (conn->nickname)
+        state = "LAUNCHED";
+      else
+        state = "NEW";
+      orconn_target_get_name(name, sizeof(name), conn);
+      smartlist_add_asprintf(status, "%s %s", name, state);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmpstart(question, "address-mappings/")) {
+    time_t min_e, max_e;
+    smartlist_t *mappings;
+    question += strlen("address-mappings/");
+    if (!strcmp(question, "all")) {
+      min_e = 0; max_e = TIME_MAX;
+    } else if (!strcmp(question, "cache")) {
+      min_e = 2; max_e = TIME_MAX;
+    } else if (!strcmp(question, "config")) {
+      min_e = 0; max_e = 0;
+    } else if (!strcmp(question, "control")) {
+      min_e = 1; max_e = 1;
+    } else {
+      return 0;
+    }
+    mappings = smartlist_new();
+    addressmap_get_mappings(mappings, min_e, max_e, 1);
+    *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp));
+    smartlist_free(mappings);
+  } else if (!strcmpstart(question, "status/")) {
+    /* Note that status/ is not a catch-all for events; there's only supposed
+     * to be a status GETINFO if there's a corresponding STATUS event. */
+    if (!strcmp(question, "status/circuit-established")) {
+      *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0");
+    } else if (!strcmp(question, "status/enough-dir-info")) {
+      *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0");
+    } else if (!strcmp(question, "status/good-server-descriptor") ||
+               !strcmp(question, "status/accepted-server-descriptor")) {
+      /* They're equivalent for now, until we can figure out how to make
+       * good-server-descriptor be what we want. See comment in
+       * control-spec.txt. */
+      *answer = tor_strdup(directories_have_accepted_server_descriptor()
+                           ? "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/or")) {
+      *answer = tor_strdup(check_whether_orport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/dir")) {
+      *answer = tor_strdup(check_whether_dirport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded")) {
+      tor_asprintf(answer, "OR=%d DIR=%d",
+                   check_whether_orport_reachable(options) ? 1 : 0,
+                   check_whether_dirport_reachable(options) ? 1 : 0);
+    } else if (!strcmp(question, "status/bootstrap-phase")) {
+      *answer = tor_strdup(last_sent_bootstrap_message);
+    } else if (!strcmpstart(question, "status/version/")) {
+      int is_server = server_mode(options);
+      networkstatus_t *c = networkstatus_get_latest_consensus();
+      version_status_t status;
+      const char *recommended;
+      if (c) {
+        recommended = is_server ? c->server_versions : c->client_versions;
+        status = tor_version_is_obsolete(VERSION, recommended);
+      } else {
+        recommended = "?";
+        status = VS_UNKNOWN;
+      }
+
+      if (!strcmp(question, "status/version/recommended")) {
+        *answer = tor_strdup(recommended);
+        return 0;
+      }
+      if (!strcmp(question, "status/version/current")) {
+        switch (status)
+          {
+          case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break;
+          case VS_OLD: *answer = tor_strdup("obsolete"); break;
+          case VS_NEW: *answer = tor_strdup("new"); break;
+          case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break;
+          case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break;
+          case VS_EMPTY: *answer = tor_strdup("none recommended"); break;
+          case VS_UNKNOWN: *answer = tor_strdup("unknown"); break;
+          default: tor_fragile_assert();
+          }
+      } else if (!strcmp(question, "status/version/num-versioning") ||
+                 !strcmp(question, "status/version/num-concurring")) {
+        tor_asprintf(answer, "%d", get_n_authorities(V3_DIRINFO));
+        log_warn(LD_GENERAL, "%s is deprecated; it no longer gives useful "
+                 "information", question);
+      }
+    } else if (!strcmp(question, "status/clients-seen")) {
+      char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL));
+      if (!bridge_stats) {
+        *errmsg = "No bridge-client stats available";
+        return -1;
+      }
+      *answer = bridge_stats;
+    } else if (!strcmp(question, "status/fresh-relay-descs")) {
+      if (!server_mode(options)) {
+        *errmsg = "Only relays have descriptors";
+        return -1;
+      }
+      routerinfo_t *r;
+      extrainfo_t *e;
+      if (router_build_fresh_descriptor(&r, &e) < 0) {
+        *errmsg = "Error generating descriptor";
+        return -1;
+      }
+      size_t size = r->cache_info.signed_descriptor_len + 1;
+      if (e) {
+        size += e->cache_info.signed_descriptor_len + 1;
+      }
+      tor_assert(r->cache_info.signed_descriptor_len);
+      char *descs = tor_malloc(size);
+      char *cp = descs;
+      memcpy(cp, signed_descriptor_get_body(&r->cache_info),
+             r->cache_info.signed_descriptor_len);
+      cp += r->cache_info.signed_descriptor_len - 1;
+      if (e) {
+        if (cp[0] == '\0') {
+          cp[0] = '\n';
+        } else if (cp[0] != '\n') {
+          cp[1] = '\n';
+          cp++;
+        }
+        memcpy(cp, signed_descriptor_get_body(&e->cache_info),
+               e->cache_info.signed_descriptor_len);
+        cp += e->cache_info.signed_descriptor_len - 1;
+      }
+      if (cp[0] == '\n') {
+        cp[0] = '\0';
+      } else if (cp[0] != '\0') {
+        cp[1] = '\0';
+      }
+      *answer = descs;
+      routerinfo_free(r);
+      extrainfo_free(e);
+    } else {
+      return 0;
+    }
+  }
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows how to enumerate hidden services
+ * created via the control port. */
+STATIC int
+getinfo_helper_onions(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  smartlist_t *onion_list = NULL;
+  (void) errmsg;  /* no errors from this method */
+
+  if (control_conn && !strcmp(question, "onions/current")) {
+    onion_list = control_conn->ephemeral_onion_services;
+  } else if (!strcmp(question, "onions/detached")) {
+    onion_list = detached_onion_services;
+  } else {
+    return 0;
+  }
+  if (!onion_list || smartlist_len(onion_list) == 0) {
+    if (answer) {
+      *answer = tor_strdup("");
+    }
+  } else {
+    if (answer) {
+      *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about network
+ * liveness. */
+static int
+getinfo_helper_liveness(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+  if (strcmp(question, "network-liveness") == 0) {
+    if (get_cached_network_liveness()) {
+      *answer = tor_strdup("up");
+    } else {
+      *answer = tor_strdup("down");
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about shared random
+ * value. */
+static int
+getinfo_helper_sr(control_connection_t *control_conn,
+                  const char *question, char **answer,
+                  const char **errmsg)
+{
+  (void) control_conn;
+  (void) errmsg;
+
+  if (!strcmp(question, "sr/current")) {
+    *answer = sr_get_current_for_control();
+  } else if (!strcmp(question, "sr/previous")) {
+    *answer = sr_get_previous_for_control();
+  }
+  /* Else statement here is unrecognized key so do nothing. */
+
+  return 0;
+}
+
+/** Callback function for GETINFO: on a given control connection, try to
+ * answer the question <b>q</b> and store the newly-allocated answer in
+ * *<b>a</b>. If an internal error occurs, return -1 and optionally set
+ * *<b>error_out</b> to point to an error message to be delivered to the
+ * controller. On success, _or if the key is not recognized_, return 0. Do not
+ * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
+ * to improve the error message.
+ */
+typedef int (*getinfo_helper_t)(control_connection_t *,
+                                const char *q, char **a,
+                                const char **error_out);
+
+/** A single item for the GETINFO question-to-answer-function table. */
+typedef struct getinfo_item_t {
+  const char *varname; /**< The value (or prefix) of the question. */
+  getinfo_helper_t fn; /**< The function that knows the answer: NULL if
+                        * this entry is documentation-only. */
+  const char *desc; /**< Description of the variable. */
+  int is_prefix; /** Must varname match exactly, or must it be a prefix? */
+} getinfo_item_t;
+
+#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 }
+#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 }
+#define DOC(name, desc) { name, NULL, desc, 0 }
+
+/** Table mapping questions accepted by GETINFO to the functions that know how
+ * to answer them. */
+static const getinfo_item_t getinfo_items[] = {
+  ITEM("version", misc, "The current version of Tor."),
+  ITEM("bw-event-cache", misc, "Cached BW events for a short interval."),
+  ITEM("config-file", misc, "Current location of the \"torrc\" file."),
+  ITEM("config-defaults-file", misc, "Current location of the defaults file."),
+  ITEM("config-text", misc,
+       "Return the string that would be written by a saveconf command."),
+  ITEM("config-can-saveconf", misc,
+       "Is it possible to save the configuration to the \"torrc\" file?"),
+  ITEM("accounting/bytes", accounting,
+       "Number of bytes read/written so far in the accounting interval."),
+  ITEM("accounting/bytes-left", accounting,
+      "Number of bytes left to write/read so far in the accounting interval."),
+  ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"),
+  ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"),
+  ITEM("accounting/interval-start", accounting,
+       "Time when the accounting period starts."),
+  ITEM("accounting/interval-end", accounting,
+       "Time when the accounting period ends."),
+  ITEM("accounting/interval-wake", accounting,
+       "Time to wake up in this accounting period."),
+  ITEM("helper-nodes", entry_guards, NULL), /* deprecated */
+  ITEM("entry-guards", entry_guards,
+       "Which nodes are we using as entry guards?"),
+  ITEM("fingerprint", misc, NULL),
+  PREFIX("config/", config, "Current configuration values."),
+  DOC("config/names",
+      "List of configuration options, types, and documentation."),
+  DOC("config/defaults",
+      "List of default values for configuration options. "
+      "See also config/names"),
+  PREFIX("current-time/", current_time, "Current time."),
+  DOC("current-time/local", "Current time on the local system."),
+  DOC("current-time/utc", "Current UTC time."),
+  PREFIX("downloads/networkstatus/", downloads,
+         "Download statuses for networkstatus objects"),
+  DOC("downloads/networkstatus/ns",
+      "Download status for current-mode networkstatus download"),
+  DOC("downloads/networkstatus/ns/bootstrap",
+      "Download status for bootstrap-time networkstatus download"),
+  DOC("downloads/networkstatus/ns/running",
+      "Download status for run-time networkstatus download"),
+  DOC("downloads/networkstatus/microdesc",
+      "Download status for current-mode microdesc download"),
+  DOC("downloads/networkstatus/microdesc/bootstrap",
+      "Download status for bootstrap-time microdesc download"),
+  DOC("downloads/networkstatus/microdesc/running",
+      "Download status for run-time microdesc download"),
+  PREFIX("downloads/cert/", downloads,
+         "Download statuses for certificates, by id fingerprint and "
+         "signing key"),
+  DOC("downloads/cert/fps",
+      "List of authority fingerprints for which any download statuses "
+      "exist"),
+  DOC("downloads/cert/fp/<fp>",
+      "Download status for <fp> with the default signing key; corresponds "
+      "to /fp/ URLs on directory server."),
+  DOC("downloads/cert/fp/<fp>/sks",
+      "List of signing keys for which specific download statuses are "
+      "available for this id fingerprint"),
+  DOC("downloads/cert/fp/<fp>/<sk>",
+      "Download status for <fp> with signing key <sk>; corresponds "
+      "to /fp-sk/ URLs on directory server."),
+  PREFIX("downloads/desc/", downloads,
+         "Download statuses for router descriptors, by descriptor digest"),
+  DOC("downloads/desc/descs",
+      "Return a list of known router descriptor digests"),
+  DOC("downloads/desc/<desc>",
+      "Return a download status for a given descriptor digest"),
+  PREFIX("downloads/bridge/", downloads,
+         "Download statuses for bridge descriptors, by bridge identity "
+         "digest"),
+  DOC("downloads/bridge/bridges",
+      "Return a list of configured bridge identity digests with download "
+      "statuses"),
+  DOC("downloads/bridge/<desc>",
+      "Return a download status for a given bridge identity digest"),
+  ITEM("info/names", misc,
+       "List of GETINFO options, types, and documentation."),
+  ITEM("events/names", misc,
+       "Events that the controller can ask for with SETEVENTS."),
+  ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"),
+  ITEM("features/names", misc, "What arguments can USEFEATURE take?"),
+  PREFIX("desc/id/", dir, "Router descriptors by ID."),
+  PREFIX("desc/name/", dir, "Router descriptors by nickname."),
+  ITEM("desc/all-recent", dir,
+       "All non-expired, non-superseded router descriptors."),
+  ITEM("desc/download-enabled", dir,
+       "Do we try to download router descriptors?"),
+  ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
+  ITEM("md/all", dir, "All known microdescriptors."),
+  PREFIX("md/id/", dir, "Microdescriptors by ID"),
+  PREFIX("md/name/", dir, "Microdescriptors by name"),
+  ITEM("md/download-enabled", dir,
+       "Do we try to download microdescriptors?"),
+  PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
+  PREFIX("hs/client/desc/id", dir,
+         "Hidden Service descriptor in client's cache by onion."),
+  PREFIX("hs/service/desc/id/", dir,
+         "Hidden Service descriptor in services's cache by onion."),
+  PREFIX("net/listeners/", listeners, "Bound addresses by type"),
+  ITEM("ns/all", networkstatus,
+       "Brief summary of router status (v2 directory format)"),
+  PREFIX("ns/id/", networkstatus,
+         "Brief summary of router status by ID (v2 directory format)."),
+  PREFIX("ns/name/", networkstatus,
+         "Brief summary of router status by nickname (v2 directory format)."),
+  PREFIX("ns/purpose/", networkstatus,
+         "Brief summary of router status by purpose (v2 directory format)."),
+  PREFIX("consensus/", networkstatus,
+         "Information about and from the ns consensus."),
+  ITEM("network-status", dir,
+       "Brief summary of router status (v1 directory format)"),
+  ITEM("network-liveness", liveness,
+       "Current opinion on whether the network is live"),
+  ITEM("circuit-status", events, "List of current circuits originating here."),
+  ITEM("stream-status", events,"List of current streams."),
+  ITEM("orconn-status", events, "A list of current OR connections."),
+  ITEM("dormant", misc,
+       "Is Tor dormant (not building circuits because it's idle)?"),
+  PREFIX("address-mappings/", events, NULL),
+  DOC("address-mappings/all", "Current address mappings."),
+  DOC("address-mappings/cache", "Current cached DNS replies."),
+  DOC("address-mappings/config",
+      "Current address mappings from configuration."),
+  DOC("address-mappings/control", "Current address mappings from controller."),
+  PREFIX("status/", events, NULL),
+  DOC("status/circuit-established",
+      "Whether we think client functionality is working."),
+  DOC("status/enough-dir-info",
+      "Whether we have enough up-to-date directory information to build "
+      "circuits."),
+  DOC("status/bootstrap-phase",
+      "The last bootstrap phase status event that Tor sent."),
+  DOC("status/clients-seen",
+      "Breakdown of client countries seen by a bridge."),
+  DOC("status/fresh-relay-descs",
+      "A fresh relay/ei descriptor pair for Tor's current state. Not stored."),
+  DOC("status/version/recommended", "List of currently recommended versions."),
+  DOC("status/version/current", "Status of the current version."),
+  DOC("status/version/num-versioning", "Number of versioning authorities."),
+  DOC("status/version/num-concurring",
+      "Number of versioning authorities agreeing on the status of the "
+      "current version"),
+  ITEM("address", misc, "IP address of this Tor host, if we can guess it."),
+  ITEM("traffic/read", misc,"Bytes read since the process was started."),
+  ITEM("traffic/written", misc,
+       "Bytes written since the process was started."),
+  ITEM("process/pid", misc, "Process id belonging to the main tor process."),
+  ITEM("process/uid", misc, "User id running the tor process."),
+  ITEM("process/user", misc,
+       "Username under which the tor process is running."),
+  ITEM("process/descriptor-limit", misc, "File descriptor limit."),
+  ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"),
+  PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."),
+  PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),
+  PREFIX("dir/status/", dir,
+         "v2 networkstatus docs as retrieved from a DirPort."),
+  ITEM("dir/status-vote/current/consensus", dir,
+       "v3 Networkstatus consensus as retrieved from a DirPort."),
+  ITEM("exit-policy/default", policies,
+       "The default value appended to the configured exit policy."),
+  ITEM("exit-policy/reject-private/default", policies,
+       "The default rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate."),
+  ITEM("exit-policy/reject-private/relay", policies,
+       "The relay-specific rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."),
+  ITEM("exit-policy/full", policies, "The entire exit policy of onion router"),
+  ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"),
+  ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"),
+  PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"),
+  ITEM("onions/current", onions,
+       "Onion services owned by the current control connection."),
+  ITEM("onions/detached", onions,
+       "Onion services detached from the control connection."),
+  ITEM("sr/current", sr, "Get current shared random value."),
+  ITEM("sr/previous", sr, "Get previous shared random value."),
+  { NULL, NULL, NULL, 0 }
+};
+
+/** Allocate and return a list of recognized GETINFO options. */
+static char *
+list_getinfo_options(void)
+{
+  int i;
+  smartlist_t *lines = smartlist_new();
+  char *ans;
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    if (!getinfo_items[i].desc)
+      continue;
+
+    smartlist_add_asprintf(lines, "%s%s -- %s\n",
+                 getinfo_items[i].varname,
+                 getinfo_items[i].is_prefix ? "*" : "",
+                 getinfo_items[i].desc);
+  }
+  smartlist_sort_strings(lines);
+
+  ans = smartlist_join_strings(lines, "", 0, NULL);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+
+  return ans;
+}
+
+/** Lookup the 'getinfo' entry <b>question</b>, and return
+ * the answer in <b>*answer</b> (or NULL if key not recognized).
+ * Return 0 if success or unrecognized, or -1 if recognized but
+ * internal error. */
+static int
+handle_getinfo_helper(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **err_out)
+{
+  int i;
+  *answer = NULL; /* unrecognized key by default */
+
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    int match;
+    if (getinfo_items[i].is_prefix)
+      match = !strcmpstart(question, getinfo_items[i].varname);
+    else
+      match = !strcmp(question, getinfo_items[i].varname);
+    if (match) {
+      tor_assert(getinfo_items[i].fn);
+      return getinfo_items[i].fn(control_conn, question, answer, err_out);
+    }
+  }
+
+  return 0; /* unrecognized */
+}
+
+/** Called when we receive a GETINFO command.  Try to fetch all requested
+ * information, and reply with information or error message. */
+static int
+handle_control_getinfo(control_connection_t *conn, uint32_t len,
+                       const char *body)
+{
+  smartlist_t *questions = smartlist_new();
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  char *ans = NULL;
+  int i;
+  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
+
+  smartlist_split_string(questions, body, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    const char *errmsg = NULL;
+
+    if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
+      if (!errmsg)
+        errmsg = "Internal error";
+      connection_printf_to_buf(conn, "551 %s\r\n", errmsg);
+      goto done;
+    }
+    if (!ans) {
+      if (errmsg) /* use provided error message */
+        smartlist_add_strdup(unrecognized, errmsg);
+      else /* use default error message */
+        smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
+    } else {
+      smartlist_add_strdup(answers, q);
+      smartlist_add(answers, ans);
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  if (smartlist_len(unrecognized)) {
+    /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
+    for (i=0; i < smartlist_len(unrecognized)-1; ++i)
+      connection_printf_to_buf(conn,
+                               "552-%s\r\n",
+                               (char *)smartlist_get(unrecognized, i));
+
+    connection_printf_to_buf(conn,
+                             "552 %s\r\n",
+                             (char *)smartlist_get(unrecognized, i));
+    goto done;
+  }
+
+  for (i = 0; i < smartlist_len(answers); i += 2) {
+    char *k = smartlist_get(answers, i);
+    char *v = smartlist_get(answers, i+1);
+    if (!strchr(v, '\n') && !strchr(v, '\r')) {
+      connection_printf_to_buf(conn, "250-%s=", k);
+      connection_write_str_to_buf(v, conn);
+      connection_write_str_to_buf("\r\n", conn);
+    } else {
+      char *esc = NULL;
+      size_t esc_len;
+      esc_len = write_escaped_data(v, strlen(v), &esc);
+      connection_printf_to_buf(conn, "250+%s=\r\n", k);
+      connection_buf_add(esc, esc_len, TO_CONN(conn));
+      tor_free(esc);
+    }
+  }
+  connection_write_str_to_buf("250 OK\r\n", conn);
+
+ done:
+  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
+  smartlist_free(answers);
+  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
+  smartlist_free(questions);
+  SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
+  smartlist_free(unrecognized);
+
+  return 0;
+}
+
+/** Given a string, convert it to a circuit purpose. */
+static uint8_t
+circuit_purpose_from_string(const char *string)
+{
+  if (!strcasecmpstart(string, "purpose="))
+    string += strlen("purpose=");
+
+  if (!strcasecmp(string, "general"))
+    return CIRCUIT_PURPOSE_C_GENERAL;
+  else if (!strcasecmp(string, "controller"))
+    return CIRCUIT_PURPOSE_CONTROLLER;
+  else
+    return CIRCUIT_PURPOSE_UNKNOWN;
+}
+
+/** Return a newly allocated smartlist containing the arguments to the command
+ * waiting in <b>body</b>. If there are fewer than <b>min_args</b> arguments,
+ * or if <b>max_args</b> is nonnegative and there are more than
+ * <b>max_args</b> arguments, send a 512 error to the controller, using
+ * <b>command</b> as the command name in the error message. */
+static smartlist_t *
+getargs_helper(const char *command, control_connection_t *conn,
+               const char *body, int min_args, int max_args)
+{
+  smartlist_t *args = smartlist_new();
+  smartlist_split_string(args, body, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  if (smartlist_len(args) < min_args) {
+    connection_printf_to_buf(conn, "512 Missing argument to %s\r\n",command);
+    goto err;
+  } else if (max_args >= 0 && smartlist_len(args) > max_args) {
+    connection_printf_to_buf(conn, "512 Too many arguments to %s\r\n",command);
+    goto err;
+  }
+  return args;
+ err:
+  SMARTLIST_FOREACH(args, char *, s, tor_free(s));
+  smartlist_free(args);
+  return NULL;
+}
+
+/** Helper.  Return the first element of <b>sl</b> at index <b>start_at</b> or
+ * higher that starts with <b>prefix</b>, case-insensitive.  Return NULL if no
+ * such element exists. */
+static const char *
+find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix)
+{
+  int i;
+  for (i = start_at; i < smartlist_len(sl); ++i) {
+    const char *elt = smartlist_get(sl, i);
+    if (!strcasecmpstart(elt, prefix))
+      return elt;
+  }
+  return NULL;
+}
+
+/** Helper.  Return true iff s is an argument that we should treat as a
+ * key-value pair. */
+static int
+is_keyval_pair(const char *s)
+{
+  /* An argument is a key-value pair if it has an =, and it isn't of the form
+   * $fingeprint=name */
+  return strchr(s, '=') && s[0] != '$';
+}
+
+/** Called when we get an EXTENDCIRCUIT message.  Try to extend the listed
+ * circuit, and report success or failure. */
+static int
+handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
+                             const char *body)
+{
+  smartlist_t *router_nicknames=NULL, *nodes=NULL;
+  origin_circuit_t *circ = NULL;
+  int zero_circ;
+  uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  smartlist_t *args;
+  (void) len;
+
+  router_nicknames = smartlist_new();
+
+  args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1);
+  if (!args)
+    goto done;
+
+  zero_circ = !strcmp("0", (char*)smartlist_get(args,0));
+
+  if (zero_circ) {
+    const char *purp = find_element_starting_with(args, 1, "PURPOSE=");
+
+    if (purp) {
+      intended_purpose = circuit_purpose_from_string(purp);
+      if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
+        SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+        smartlist_free(args);
+        goto done;
+      }
+    }
+
+    if ((smartlist_len(args) == 1) ||
+        (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) {
+      // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar"
+      circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY);
+      if (!circ) {
+        connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
+      } else {
+        connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
+                  (unsigned long)circ->global_identifier);
+      }
+      SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+      smartlist_free(args);
+      goto done;
+    }
+    // "EXTENDCIRCUIT 0 router1,router2" ||
+    // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo"
+  }
+
+  if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) {
+    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
+                             (char*)smartlist_get(args, 0));
+    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+    smartlist_free(args);
+    goto done;
+  }
+
+  if (smartlist_len(args) < 2) {
+    connection_printf_to_buf(conn,
+                             "512 syntax error: not enough arguments.\r\n");
+    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+    smartlist_free(args);
+    goto done;
+  }
+
+  smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0);
+
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+
+  nodes = smartlist_new();
+  int first_node = zero_circ;
+  SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
+    const node_t *node = node_get_by_nickname(n, 0);
+    if (!node) {
+      connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n);
+      goto done;
+    }
+    if (!node_has_preferred_descriptor(node, first_node)) {
+      connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n);
+      goto done;
+    }
+    smartlist_add(nodes, (void*)node);
+    first_node = 0;
+  } SMARTLIST_FOREACH_END(n);
+  if (!smartlist_len(nodes)) {
+    connection_write_str_to_buf("512 No router names provided\r\n", conn);
+    goto done;
+  }
+
+  if (zero_circ) {
+    /* start a new circuit */
+    circ = origin_circuit_init(intended_purpose, 0);
+  }
+
+  /* now circ refers to something that is ready to be extended */
+  first_node = zero_circ;
+  SMARTLIST_FOREACH(nodes, const node_t *, node,
+  {
+    extend_info_t *info = extend_info_from_node(node, first_node);
+    if (!info) {
+      tor_assert_nonfatal(first_node);
+      log_warn(LD_CONTROL,
+               "controller tried to connect to a node that lacks a suitable "
+               "descriptor, or which doesn't have any "
+               "addresses that are allowed by the firewall configuration; "
+               "circuit marked for closing.");
+      circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
+      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
+      goto done;
+    }
+    circuit_append_new_exit(circ, info);
+    if (circ->build_state->desired_path_len > 1) {
+      circ->build_state->onehop_tunnel = 0;
+    }
+    extend_info_free(info);
+    first_node = 0;
+  });
+
+  /* now that we've populated the cpath, start extending */
+  if (zero_circ) {
+    int err_reason = 0;
+    if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
+      circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
+      goto done;
+    }
+  } else {
+    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
+        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
+      int err_reason = 0;
+      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
+      if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
+        log_info(LD_CONTROL,
+                 "send_next_onion_skin failed; circuit marked for closing.");
+        circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+        connection_write_str_to_buf("551 Couldn't send onion skin\r\n", conn);
+        goto done;
+      }
+    }
+  }
+
+  connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
+                             (unsigned long)circ->global_identifier);
+  if (zero_circ) /* send a 'launched' event, for completeness */
+    control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0);
+ done:
+  SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n));
+  smartlist_free(router_nicknames);
+  smartlist_free(nodes);
+  return 0;
+}
+
+/** Called when we get a SETCIRCUITPURPOSE message. If we can find the
+ * circuit and it's a valid purpose, change it. */
+static int
+handle_control_setcircuitpurpose(control_connection_t *conn,
+                                 uint32_t len, const char *body)
+{
+  origin_circuit_t *circ = NULL;
+  uint8_t new_purpose;
+  smartlist_t *args;
+  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
+
+  args = getargs_helper("SETCIRCUITPURPOSE", conn, body, 2, -1);
+  if (!args)
+    goto done;
+
+  if (!(circ = get_circ(smartlist_get(args,0)))) {
+    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
+                             (char*)smartlist_get(args, 0));
+    goto done;
+  }
+
+  {
+    const char *purp = find_element_starting_with(args,1,"PURPOSE=");
+    if (!purp) {
+      connection_write_str_to_buf("552 No purpose given\r\n", conn);
+      goto done;
+    }
+    new_purpose = circuit_purpose_from_string(purp);
+    if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+      connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
+      goto done;
+    }
+  }
+
+  circuit_change_purpose(TO_CIRCUIT(circ), new_purpose);
+  connection_write_str_to_buf("250 OK\r\n", conn);
+
+ done:
+  if (args) {
+    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+    smartlist_free(args);
+  }
+  return 0;
+}
+
+/** Called when we get an ATTACHSTREAM message.  Try to attach the requested
+ * stream, and report success or failure. */
+static int
+handle_control_attachstream(control_connection_t *conn, uint32_t len,
+                            const char *body)
+{
+  entry_connection_t *ap_conn = NULL;
+  origin_circuit_t *circ = NULL;
+  int zero_circ;
+  smartlist_t *args;
+  crypt_path_t *cpath=NULL;
+  int hop=0, hop_line_ok=1;
+  (void) len;
+
+  args = getargs_helper("ATTACHSTREAM", conn, body, 2, -1);
+  if (!args)
+    return 0;
+
+  zero_circ = !strcmp("0", (char*)smartlist_get(args,1));
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0)))) {
+    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
+                             (char*)smartlist_get(args, 0));
+  } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) {
+    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
+                             (char*)smartlist_get(args, 1));
+  } else if (circ) {
+    const char *hopstring = find_element_starting_with(args,2,"HOP=");
+    if (hopstring) {
+      hopstring += strlen("HOP=");
+      hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX,
+                                  &hop_line_ok, NULL);
+      if (!hop_line_ok) { /* broken hop line */
+        connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", hopstring);
+      }
+    }
+  }
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  if (!ap_conn || (!zero_circ && !circ) || !hop_line_ok)
+    return 0;
+
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) {
+    connection_write_str_to_buf(
+                    "555 Connection is not managed by controller.\r\n",
+                    conn);
+    return 0;
+  }
+
+  /* Do we need to detach it first? */
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) {
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn);
+    circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn);
+    connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT);
+    /* Un-mark it as ending, since we're going to reuse it. */
+    edge_conn->edge_has_sent_end = 0;
+    edge_conn->end_reason = 0;
+    if (tmpcirc)
+      circuit_detach_stream(tmpcirc, edge_conn);
+    CONNECTION_AP_EXPECT_NONPENDING(ap_conn);
+    TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT;
+  }
+
+  if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) {
+    connection_write_str_to_buf(
+                    "551 Can't attach stream to non-open origin circuit\r\n",
+                    conn);
+    return 0;
+  }
+  /* Is this a single hop circuit? */
+  if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
+    connection_write_str_to_buf(
+               "551 Can't attach stream to this one-hop circuit.\r\n", conn);
+    return 0;
+  }
+
+  if (circ && hop>0) {
+    /* find this hop in the circuit, and set cpath */
+    cpath = circuit_get_cpath_hop(circ, hop);
+    if (!cpath) {
+      connection_printf_to_buf(conn,
+                               "551 Circuit doesn't have %d hops.\r\n", hop);
+      return 0;
+    }
+  }
+  if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) {
+    connection_write_str_to_buf("551 Unable to attach stream\r\n", conn);
+    return 0;
+  }
+  send_control_done(conn);
+  return 0;
+}
+
+/** Called when we get a POSTDESCRIPTOR message.  Try to learn the provided
+ * descriptor, and report success or failure. */
+static int
+handle_control_postdescriptor(control_connection_t *conn, uint32_t len,
+                              const char *body)
+{
+  char *desc;
+  const char *msg=NULL;
+  uint8_t purpose = ROUTER_PURPOSE_GENERAL;
+  int cache = 0; /* eventually, we may switch this to 1 */
+
+  const char *cp = memchr(body, '\n', len);
+
+  if (cp == NULL) {
+    connection_printf_to_buf(conn, "251 Empty body\r\n");
+    return 0;
+  }
+  ++cp;
+
+  char *cmdline = tor_memdup_nulterm(body, cp-body);
+  smartlist_t *args = smartlist_new();
+  smartlist_split_string(args, cmdline, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  SMARTLIST_FOREACH_BEGIN(args, char *, option) {
+    if (!strcasecmpstart(option, "purpose=")) {
+      option += strlen("purpose=");
+      purpose = router_purpose_from_string(option);
+      if (purpose == ROUTER_PURPOSE_UNKNOWN) {
+        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n",
+                                 option);
+        goto done;
+      }
+    } else if (!strcasecmpstart(option, "cache=")) {
+      option += strlen("cache=");
+      if (!strcasecmp(option, "no"))
+        cache = 0;
+      else if (!strcasecmp(option, "yes"))
+        cache = 1;
+      else {
+        connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n",
+                                 option);
+        goto done;
+      }
+    } else { /* unrecognized argument? */
+      connection_printf_to_buf(conn,
+        "512 Unexpected argument \"%s\" to postdescriptor\r\n", option);
+      goto done;
+    }
+  } SMARTLIST_FOREACH_END(option);
+
+  read_escaped_data(cp, len-(cp-body), &desc);
+
+  switch (router_load_single_router(desc, purpose, cache, &msg)) {
+  case -1:
+    if (!msg) msg = "Could not parse descriptor";
+    connection_printf_to_buf(conn, "554 %s\r\n", msg);
+    break;
+  case 0:
+    if (!msg) msg = "Descriptor not added";
+    connection_printf_to_buf(conn, "251 %s\r\n",msg);
+    break;
+  case 1:
+    send_control_done(conn);
+    break;
+  }
+
+  tor_free(desc);
+ done:
+  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
+  smartlist_free(args);
+  tor_free(cmdline);
+  return 0;
+}
+
+/** Called when we receive a REDIRECTSTERAM command.  Try to change the target
+ * address of the named AP stream, and report success or failure. */
+static int
+handle_control_redirectstream(control_connection_t *conn, uint32_t len,
+                              const char *body)
+{
+  entry_connection_t *ap_conn = NULL;
+  char *new_addr = NULL;
+  uint16_t new_port = 0;
+  smartlist_t *args;
+  (void) len;
+
+  args = getargs_helper("REDIRECTSTREAM", conn, body, 2, -1);
+  if (!args)
+    return 0;
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0)))
+           || !ap_conn->socks_request) {
+    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
+                             (char*)smartlist_get(args, 0));
+  } else {
+    int ok = 1;
+    if (smartlist_len(args) > 2) { /* they included a port too */
+      new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2),
+                                            10, 1, 65535, &ok, NULL);
+    }
+    if (!ok) {
+      connection_printf_to_buf(conn, "512 Cannot parse port \"%s\"\r\n",
+                               (char*)smartlist_get(args, 2));
+    } else {
+      new_addr = tor_strdup(smartlist_get(args, 1));
+    }
+  }
+
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  if (!new_addr)
+    return 0;
+
+  strlcpy(ap_conn->socks_request->address, new_addr,
+          sizeof(ap_conn->socks_request->address));
+  if (new_port)
+    ap_conn->socks_request->port = new_port;
+  tor_free(new_addr);
+  send_control_done(conn);
+  return 0;
+}
+
+/** Called when we get a CLOSESTREAM command; try to close the named stream
+ * and report success or failure. */
+static int
+handle_control_closestream(control_connection_t *conn, uint32_t len,
+                           const char *body)
+{
+  entry_connection_t *ap_conn=NULL;
+  uint8_t reason=0;
+  smartlist_t *args;
+  int ok;
+  (void) len;
+
+  args = getargs_helper("CLOSESTREAM", conn, body, 2, -1);
+  if (!args)
+    return 0;
+
+  else if (!(ap_conn = get_stream(smartlist_get(args, 0))))
+    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
+                             (char*)smartlist_get(args, 0));
+  else {
+    reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255,
+                                       &ok, NULL);
+    if (!ok) {
+      connection_printf_to_buf(conn, "552 Unrecognized reason \"%s\"\r\n",
+                               (char*)smartlist_get(args, 1));
+      ap_conn = NULL;
+    }
+  }
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  if (!ap_conn)
+    return 0;
+
+  connection_mark_unattached_ap(ap_conn, reason);
+  send_control_done(conn);
+  return 0;
+}
+
+/** Called when we get a CLOSECIRCUIT command; try to close the named circuit
+ * and report success or failure. */
+static int
+handle_control_closecircuit(control_connection_t *conn, uint32_t len,
+                            const char *body)
+{
+  origin_circuit_t *circ = NULL;
+  int safe = 0;
+  smartlist_t *args;
+  (void) len;
+
+  args = getargs_helper("CLOSECIRCUIT", conn, body, 1, -1);
+  if (!args)
+    return 0;
+
+  if (!(circ=get_circ(smartlist_get(args, 0))))
+    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
+                             (char*)smartlist_get(args, 0));
+  else {
+    int i;
+    for (i=1; i < smartlist_len(args); ++i) {
+      if (!strcasecmp(smartlist_get(args, i), "IfUnused"))
+        safe = 1;
+      else
+        log_info(LD_CONTROL, "Skipping unknown option %s",
+                 (char*)smartlist_get(args,i));
+    }
+  }
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  if (!circ)
+    return 0;
+
+  if (!safe || !circ->p_streams) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED);
+  }
+
+  send_control_done(conn);
+  return 0;
+}
+
+/** Called when we get a RESOLVE command: start trying to resolve
+ * the listed addresses. */
+static int
+handle_control_resolve(control_connection_t *conn, uint32_t len,
+                       const char *body)
+{
+  smartlist_t *args, *failed;
+  int is_reverse = 0;
+  (void) len; /* body is nul-terminated; it's safe to ignore the length */
+
+  if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) {
+    log_warn(LD_CONTROL, "Controller asked us to resolve an address, but "
+             "isn't listening for ADDRMAP events.  It probably won't see "
+             "the answer.");
+  }
+  args = smartlist_new();
+  smartlist_split_string(args, body, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  {
+    const char *modearg = find_element_starting_with(args, 0, "mode=");
+    if (modearg && !strcasecmp(modearg, "mode=reverse"))
+      is_reverse = 1;
+  }
+  failed = smartlist_new();
+  SMARTLIST_FOREACH(args, const char *, arg, {
+      if (!is_keyval_pair(arg)) {
+          if (dnsserv_launch_request(arg, is_reverse, conn)<0)
+            smartlist_add(failed, (char*)arg);
+      }
+  });
+
+  send_control_done(conn);
+  SMARTLIST_FOREACH(failed, const char *, arg, {
+      control_event_address_mapped(arg, arg, time(NULL),
+                                   "internal", 0);
+  });
+
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  smartlist_free(failed);
+  return 0;
+}
+
+/** Called when we get a PROTOCOLINFO command: send back a reply. */
+static int
+handle_control_protocolinfo(control_connection_t *conn, uint32_t len,
+                            const char *body)
+{
+  const char *bad_arg = NULL;
+  smartlist_t *args;
+  (void)len;
+
+  conn->have_sent_protocolinfo = 1;
+  args = smartlist_new();
+  smartlist_split_string(args, body, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  SMARTLIST_FOREACH(args, const char *, arg, {
+      int ok;
+      tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL);
+      if (!ok) {
+        bad_arg = arg;
+        break;
+      }
+    });
+  if (bad_arg) {
+    connection_printf_to_buf(conn, "513 No such version %s\r\n",
+                             escaped(bad_arg));
+    /* Don't tolerate bad arguments when not authenticated. */
+    if (!STATE_IS_OPEN(TO_CONN(conn)->state))
+      connection_mark_for_close(TO_CONN(conn));
+    goto done;
+  } else {
+    const or_options_t *options = get_options();
+    int cookies = options->CookieAuthentication;
+    char *cfile = get_controller_cookie_file_name();
+    char *abs_cfile;
+    char *esc_cfile;
+    char *methods;
+    abs_cfile = make_path_absolute(cfile);
+    esc_cfile = esc_for_log(abs_cfile);
+    {
+      int passwd = (options->HashedControlPassword != NULL ||
+                    options->HashedControlSessionPassword != NULL);
+      smartlist_t *mlist = smartlist_new();
+      if (cookies) {
+        smartlist_add(mlist, (char*)"COOKIE");
+        smartlist_add(mlist, (char*)"SAFECOOKIE");
+      }
+      if (passwd)
+        smartlist_add(mlist, (char*)"HASHEDPASSWORD");
+      if (!cookies && !passwd)
+        smartlist_add(mlist, (char*)"NULL");
+      methods = smartlist_join_strings(mlist, ",", 0, NULL);
+      smartlist_free(mlist);
+    }
+
+    connection_printf_to_buf(conn,
+                             "250-PROTOCOLINFO 1\r\n"
+                             "250-AUTH METHODS=%s%s%s\r\n"
+                             "250-VERSION Tor=%s\r\n"
+                             "250 OK\r\n",
+                             methods,
+                             cookies?" COOKIEFILE=":"",
+                             cookies?esc_cfile:"",
+                             escaped(VERSION));
+    tor_free(methods);
+    tor_free(cfile);
+    tor_free(abs_cfile);
+    tor_free(esc_cfile);
+  }
+ done:
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  return 0;
+}
+
+/** Called when we get an AUTHCHALLENGE command. */
+static int
+handle_control_authchallenge(control_connection_t *conn, uint32_t len,
+                             const char *body)
+{
+  const char *cp = body;
+  char *client_nonce;
+  size_t client_nonce_len;
+  char server_hash[DIGEST256_LEN];
+  char server_hash_encoded[HEX_DIGEST256_LEN+1];
+  char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN];
+  char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1];
+
+  cp += strspn(cp, " \t\n\r");
+  if (!strcasecmpstart(cp, "SAFECOOKIE")) {
+    cp += strlen("SAFECOOKIE");
+  } else {
+    connection_write_str_to_buf("513 AUTHCHALLENGE only supports SAFECOOKIE "
+                                "authentication\r\n", conn);
+    connection_mark_for_close(TO_CONN(conn));
+    return -1;
+  }
+
+  if (!authentication_cookie_is_set) {
+    connection_write_str_to_buf("515 Cookie authentication is disabled\r\n",
+                                conn);
+    connection_mark_for_close(TO_CONN(conn));
+    return -1;
+  }
+
+  cp += strspn(cp, " \t\n\r");
+  if (*cp == '"') {
+    const char *newcp =
+      decode_escaped_string(cp, len - (cp - body),
+                            &client_nonce, &client_nonce_len);
+    if (newcp == NULL) {
+      connection_write_str_to_buf("513 Invalid quoted client nonce\r\n",
+                                  conn);
+      connection_mark_for_close(TO_CONN(conn));
+      return -1;
+    }
+    cp = newcp;
+  } else {
+    size_t client_nonce_encoded_len = strspn(cp, "0123456789ABCDEFabcdef");
+
+    client_nonce_len = client_nonce_encoded_len / 2;
+    client_nonce = tor_malloc_zero(client_nonce_len);
+
+    if (base16_decode(client_nonce, client_nonce_len,
+                      cp, client_nonce_encoded_len)
+                      != (int) client_nonce_len) {
+      connection_write_str_to_buf("513 Invalid base16 client nonce\r\n",
+                                  conn);
+      connection_mark_for_close(TO_CONN(conn));
+      tor_free(client_nonce);
+      return -1;
+    }
+
+    cp += client_nonce_encoded_len;
+  }
+
+  cp += strspn(cp, " \t\n\r");
+  if (*cp != '\0' ||
+      cp != body + len) {
+    connection_write_str_to_buf("513 Junk at end of AUTHCHALLENGE command\r\n",
+                                conn);
+    connection_mark_for_close(TO_CONN(conn));
+    tor_free(client_nonce);
+    return -1;
+  }
+  crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+  /* Now compute and send the server-to-controller response, and the
+   * server's nonce. */
+  tor_assert(authentication_cookie != NULL);
+
+  {
+    size_t tmp_len = (AUTHENTICATION_COOKIE_LEN +
+                      client_nonce_len +
+                      SAFECOOKIE_SERVER_NONCE_LEN);
+    char *tmp = tor_malloc_zero(tmp_len);
+    char *client_hash = tor_malloc_zero(DIGEST256_LEN);
+    memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len,
+           server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+    crypto_hmac_sha256(server_hash,
+                       SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT,
+                       strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    crypto_hmac_sha256(client_hash,
+                       SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT,
+                       strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    conn->safecookie_client_hash = client_hash;
+
+    tor_free(tmp);
+  }
+
+  base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
+                server_hash, sizeof(server_hash));
+  base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
+                server_nonce, sizeof(server_nonce));
+
+  connection_printf_to_buf(conn,
+                           "250 AUTHCHALLENGE SERVERHASH=%s "
+                           "SERVERNONCE=%s\r\n",
+                           server_hash_encoded,
+                           server_nonce_encoded);
+
+  tor_free(client_nonce);
+  return 0;
+}
+
+/** Called when we get a USEFEATURE command: parse the feature list, and
+ * set up the control_connection's options properly. */
+static int
+handle_control_usefeature(control_connection_t *conn,
+                          uint32_t len,
+                          const char *body)
+{
+  smartlist_t *args;
+  int bad = 0;
+  (void) len; /* body is nul-terminated; it's safe to ignore the length */
+  args = smartlist_new();
+  smartlist_split_string(args, body, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
+      if (!strcasecmp(arg, "VERBOSE_NAMES"))
+        ;
+      else if (!strcasecmp(arg, "EXTENDED_EVENTS"))
+        ;
+      else {
+        connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n",
+                                 arg);
+        bad = 1;
+        break;
+      }
+  } SMARTLIST_FOREACH_END(arg);
+
+  if (!bad) {
+    send_control_done(conn);
+  }
+
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  return 0;
+}
+
+/** Implementation for the DROPGUARDS command. */
+static int
+handle_control_dropguards(control_connection_t *conn,
+                          uint32_t len,
+                          const char *body)
+{
+  smartlist_t *args;
+  (void) len; /* body is nul-terminated; it's safe to ignore the length */
+  args = smartlist_new();
+  smartlist_split_string(args, body, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+  static int have_warned = 0;
+  if (! have_warned) {
+    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
+             "the risks before using it. It may be removed in a future "
+             "version of Tor.");
+    have_warned = 1;
+  }
+
+  if (smartlist_len(args)) {
+    connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n");
+  } else {
+    remove_all_entry_guards();
+    send_control_done(conn);
+  }
+
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  return 0;
+}
+
+/** Implementation for the HSFETCH command. */
+static int
+handle_control_hsfetch(control_connection_t *conn, uint32_t len,
+                       const char *body)
+{
+  int i;
+  char digest[DIGEST_LEN], *hsaddress = NULL, *arg1 = NULL, *desc_id = NULL;
+  smartlist_t *args = NULL, *hsdirs = NULL;
+  (void) len; /* body is nul-terminated; it's safe to ignore the length */
+  static const char *hsfetch_command = "HSFETCH";
+  static const char *v2_str = "v2-";
+  const size_t v2_str_len = strlen(v2_str);
+  rend_data_t *rend_query = NULL;
+
+  /* Make sure we have at least one argument, the HSAddress. */
+  args = getargs_helper(hsfetch_command, conn, body, 1, -1);
+  if (!args) {
+    goto exit;
+  }
+
+  /* Extract the first argument (either HSAddress or DescID). */
+  arg1 = smartlist_get(args, 0);
+  /* Test if it's an HS address without the .onion part. */
+  if (rend_valid_v2_service_id(arg1)) {
+    hsaddress = arg1;
+  } else if (strcmpstart(arg1, v2_str) == 0 &&
+             rend_valid_descriptor_id(arg1 + v2_str_len) &&
+             base32_decode(digest, sizeof(digest), arg1 + v2_str_len,
+                           REND_DESC_ID_V2_LEN_BASE32) == 0) {
+    /* We have a well formed version 2 descriptor ID. Keep the decoded value
+     * of the id. */
+    desc_id = digest;
+  } else {
+    connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n",
+                             arg1);
+    goto done;
+  }
+
+  static const char *opt_server = "SERVER=";
+
+  /* Skip first argument because it's the HSAddress or DescID. */
+  for (i = 1; i < smartlist_len(args); ++i) {
+    const char *arg = smartlist_get(args, i);
+    const node_t *node;
+
+    if (!strcasecmpstart(arg, opt_server)) {
+      const char *server;
+
+      server = arg + strlen(opt_server);
+      node = node_get_by_hex_id(server, 0);
+      if (!node) {
+        connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
+                                 server);
+        goto done;
+      }
+      if (!hsdirs) {
+        /* Stores routerstatus_t object for each specified server. */
+        hsdirs = smartlist_new();
+      }
+      /* Valid server, add it to our local list. */
+      smartlist_add(hsdirs, node->rs);
+    } else {
+      connection_printf_to_buf(conn, "513 Unexpected argument \"%s\"\r\n",
+                               arg);
+      goto done;
+    }
+  }
+
+  rend_query = rend_data_client_create(hsaddress, desc_id, NULL,
+                                       REND_NO_AUTH);
+  if (rend_query == NULL) {
+    connection_printf_to_buf(conn, "551 Error creating the HS query\r\n");
+    goto done;
+  }
+
+  /* Using a descriptor ID, we force the user to provide at least one
+   * hsdir server using the SERVER= option. */
+  if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) {
+      connection_printf_to_buf(conn, "512 %s option is required\r\n",
+                               opt_server);
+      goto done;
+  }
+
+  /* We are about to trigger HSDir fetch so send the OK now because after
+   * that 650 event(s) are possible so better to have the 250 OK before them
+   * to avoid out of order replies. */
+  send_control_done(conn);
+
+  /* Trigger the fetch using the built rend query and possibly a list of HS
+   * directory to use. This function ignores the client cache thus this will
+   * always send a fetch command. */
+  rend_client_fetch_v2_desc(rend_query, hsdirs);
+
+ done:
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  /* Contains data pointer that we don't own thus no cleanup. */
+  smartlist_free(hsdirs);
+  rend_data_free(rend_query);
+ exit:
+  return 0;
+}
+
+/** Implementation for the HSPOST command. */
+static int
+handle_control_hspost(control_connection_t *conn,
+                      uint32_t len,
+                      const char *body)
+{
+  static const char *opt_server = "SERVER=";
+  static const char *opt_hsaddress = "HSADDRESS=";
+  smartlist_t *hs_dirs = NULL;
+  const char *encoded_desc = body;
+  size_t encoded_desc_len = len;
+  const char *onion_address = NULL;
+
+  char *cp = memchr(body, '\n', len);
+  if (cp == NULL) {
+    connection_printf_to_buf(conn, "251 Empty body\r\n");
+    return 0;
+  }
+  char *argline = tor_strndup(body, cp-body);
+
+  smartlist_t *args = smartlist_new();
+
+  /* If any SERVER= or HSADDRESS= options were specified, try to parse
+   * the options line. */
+  if (!strcasecmpstart(argline, opt_server) ||
+      !strcasecmpstart(argline, opt_hsaddress)) {
+    /* encoded_desc begins after a newline character */
+    cp = cp + 1;
+    encoded_desc = cp;
+    encoded_desc_len = len-(cp-body);
+
+    smartlist_split_string(args, argline, " ",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+    SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
+      if (!strcasecmpstart(arg, opt_server)) {
+        const char *server = arg + strlen(opt_server);
+        const node_t *node = node_get_by_hex_id(server, 0);
+
+        if (!node || !node->rs) {
+          connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
+                                   server);
+          goto done;
+        }
+        /* Valid server, add it to our local list. */
+        if (!hs_dirs)
+          hs_dirs = smartlist_new();
+        smartlist_add(hs_dirs, node->rs);
+      } else if (!strcasecmpstart(arg, opt_hsaddress)) {
+        const char *address = arg + strlen(opt_hsaddress);
+        if (!hs_address_is_valid(address)) {
+          connection_printf_to_buf(conn, "512 Malformed onion address\r\n");
+          goto done;
+        }
+        onion_address = address;
+      } else {
+        connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n",
+                                 arg);
+        goto done;
+      }
+    } SMARTLIST_FOREACH_END(arg);
+  }
+
+  /* Handle the v3 case. */
+  if (onion_address) {
+    char *desc_str = NULL;
+    read_escaped_data(encoded_desc, encoded_desc_len, &desc_str);
+    if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) {
+      connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
+    } else {
+      send_control_done(conn);
+    }
+    tor_free(desc_str);
+    goto done;
+  }
+
+  /* From this point on, it is only v2. */
+
+  /* Read the dot encoded descriptor, and parse it. */
+  rend_encoded_v2_service_descriptor_t *desc =
+      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
+  read_escaped_data(encoded_desc, encoded_desc_len, &desc->desc_str);
+
+  rend_service_descriptor_t *parsed = NULL;
+  char *intro_content = NULL;
+  size_t intro_size;
+  size_t encoded_size;
+  const char *next_desc;
+  if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content,
+                                        &intro_size, &encoded_size,
+                                        &next_desc, desc->desc_str, 1)) {
+    /* Post the descriptor. */
+    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+    if (!rend_get_service_id(parsed->pk, serviceid)) {
+      smartlist_t *descs = smartlist_new();
+      smartlist_add(descs, desc);
+
+      /* We are about to trigger HS descriptor upload so send the OK now
+       * because after that 650 event(s) are possible so better to have the
+       * 250 OK before them to avoid out of order replies. */
+      send_control_done(conn);
+
+      /* Trigger the descriptor upload */
+      directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0);
+      smartlist_free(descs);
+    }
+
+    rend_service_descriptor_free(parsed);
+  } else {
+    connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
+  }
+
+  tor_free(intro_content);
+  rend_encoded_v2_service_descriptor_free(desc);
+ done:
+  tor_free(argline);
+  smartlist_free(hs_dirs); /* Contents belong to the rend service code. */
+  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
+  smartlist_free(args);
+  return 0;
+}
+
+/* Helper function for ADD_ONION that adds an ephemeral service depending on
+ * the given hs_version.
+ *
+ * The secret key in pk depends on the hs_version. The ownership of the key
+ * used in pk is given to the HS subsystem so the caller must stop accessing
+ * it after.
+ *
+ * The port_cfgs is a list of service port. Ownership transferred to service.
+ * The max_streams refers to the MaxStreams= key.
+ * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
+ * The auth_type is the authentication type of the clients in auth_clients.
+ * The ownership of that list is transferred to the service.
+ *
+ * On success (RSAE_OKAY), the address_out points to a newly allocated string
+ * containing the onion address without the .onion part. On error, address_out
+ * is untouched. */
+static hs_service_add_ephemeral_status_t
+add_onion_helper_add_service(int hs_version,
+                             add_onion_secret_key_t *pk,
+                             smartlist_t *port_cfgs, int max_streams,
+                             int max_streams_close_circuit, int auth_type,
+                             smartlist_t *auth_clients, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+
+  tor_assert(pk);
+  tor_assert(port_cfgs);
+  tor_assert(address_out);
+
+  switch (hs_version) {
+  case HS_VERSION_TWO:
+    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
+                                     max_streams_close_circuit, auth_type,
+                                     auth_clients, address_out);
+    break;
+  case HS_VERSION_THREE:
+    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
+                                   max_streams_close_circuit, address_out);
+    break;
+  default:
+    tor_assert_unreached();
+  }
+
+  return ret;
+}
+
+/** Called when we get a ADD_ONION command; parse the body, and set up
+ * the new ephemeral Onion Service. */
+static int
+handle_control_add_onion(control_connection_t *conn,
+                         uint32_t len,
+                         const char *body)
+{
+  smartlist_t *args;
+  int arg_len;
+  (void) len; /* body is nul-terminated; it's safe to ignore the length */
+  args = getargs_helper("ADD_ONION", conn, body, 2, -1);
+  if (!args)
+    return 0;
+  arg_len = smartlist_len(args);
+
+  /* Parse all of the arguments that do not involve handling cryptographic
+   * material first, since there's no reason to touch that at all if any of
+   * the other arguments are malformed.
+   */
+  smartlist_t *port_cfgs = smartlist_new();
+  smartlist_t *auth_clients = NULL;
+  smartlist_t *auth_created_clients = NULL;
+  int discard_pk = 0;
+  int detach = 0;
+  int max_streams = 0;
+  int max_streams_close_circuit = 0;
+  rend_auth_type_t auth_type = REND_NO_AUTH;
+  /* Default to adding an anonymous hidden service if no flag is given */
+  int non_anonymous = 0;
+  for (int i = 1; i < arg_len; i++) {
+    static const char *port_prefix = "Port=";
+    static const char *flags_prefix = "Flags=";
+    static const char *max_s_prefix = "MaxStreams=";
+    static const char *auth_prefix = "ClientAuth=";
+
+    const char *arg = smartlist_get(args, (int)i);
+    if (!strcasecmpstart(arg, port_prefix)) {
+      /* "Port=VIRTPORT[,TARGET]". */
+      const char *port_str = arg + strlen(port_prefix);
+
+      rend_service_port_config_t *cfg =
+          rend_service_parse_port_config(port_str, ",", NULL);
+      if (!cfg) {
+        connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
+        goto out;
+      }
+      smartlist_add(port_cfgs, cfg);
+    } else if (!strcasecmpstart(arg, max_s_prefix)) {
+      /* "MaxStreams=[0..65535]". */
+      const char *max_s_str = arg + strlen(max_s_prefix);
+      int ok = 0;
+      max_streams = (int)tor_parse_long(max_s_str, 10, 0, 65535, &ok, NULL);
+      if (!ok) {
+        connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n");
+        goto out;
+      }
+    } else if (!strcasecmpstart(arg, flags_prefix)) {
+      /* "Flags=Flag[,Flag]", where Flag can be:
+       *   * 'DiscardPK' - If tor generates the keypair, do not include it in
+       *                   the response.
+       *   * 'Detach' - Do not tie this onion service to any particular control
+       *                connection.
+       *   * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is
+       *                                exceeded.
+       *   * 'BasicAuth' - Client authorization using the 'basic' method.
+       *   * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this
+       *                      flag is present, tor must be in non-anonymous
+       *                      hidden service mode. If this flag is absent,
+       *                      tor must be in anonymous hidden service mode.
+       */
+      static const char *discard_flag = "DiscardPK";
+      static const char *detach_flag = "Detach";
+      static const char *max_s_close_flag = "MaxStreamsCloseCircuit";
+      static const char *basicauth_flag = "BasicAuth";
+      static const char *non_anonymous_flag = "NonAnonymous";
+
+      smartlist_t *flags = smartlist_new();
+      int bad = 0;
+
+      smartlist_split_string(flags, arg + strlen(flags_prefix), ",",
+                             SPLIT_IGNORE_BLANK, 0);
+      if (smartlist_len(flags) < 1) {
+        connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n");
+        bad = 1;
+      }
+      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag)
+      {
+        if (!strcasecmp(flag, discard_flag)) {
+          discard_pk = 1;
+        } else if (!strcasecmp(flag, detach_flag)) {
+          detach = 1;
+        } else if (!strcasecmp(flag, max_s_close_flag)) {
+          max_streams_close_circuit = 1;
+        } else if (!strcasecmp(flag, basicauth_flag)) {
+          auth_type = REND_BASIC_AUTH;
+        } else if (!strcasecmp(flag, non_anonymous_flag)) {
+          non_anonymous = 1;
+        } else {
+          connection_printf_to_buf(conn,
+                                   "512 Invalid 'Flags' argument: %s\r\n",
+                                   escaped(flag));
+          bad = 1;
+          break;
+        }
+      } SMARTLIST_FOREACH_END(flag);
+      SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
+      smartlist_free(flags);
+      if (bad)
+        goto out;
+    } else if (!strcasecmpstart(arg, auth_prefix)) {
+      char *err_msg = NULL;
+      int created = 0;
+      rend_authorized_client_t *client =
+        add_onion_helper_clientauth(arg + strlen(auth_prefix),
+                                    &created, &err_msg);
+      if (!client) {
+        if (err_msg) {
+          connection_write_str_to_buf(err_msg, conn);
+          tor_free(err_msg);
+        }
+        goto out;
+      }
+
+      if (auth_clients != NULL) {
+        int bad = 0;
+        SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) {
+          if (strcmp(ac->client_name, client->client_name) == 0) {
+            bad = 1;
+            break;
+          }
+        } SMARTLIST_FOREACH_END(ac);
+        if (bad) {
+          connection_printf_to_buf(conn,
+                                   "512 Duplicate name in ClientAuth\r\n");
+          rend_authorized_client_free(client);
+          goto out;
+        }
+      } else {
+        auth_clients = smartlist_new();
+        auth_created_clients = smartlist_new();
+      }
+      smartlist_add(auth_clients, client);
+      if (created) {
+        smartlist_add(auth_created_clients, client);
+      }
+    } else {
+      connection_printf_to_buf(conn, "513 Invalid argument\r\n");
+      goto out;
+    }
+  }
+  if (smartlist_len(port_cfgs) == 0) {
+    connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n");
+    goto out;
+  } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) {
+    connection_printf_to_buf(conn, "512 No auth type specified\r\n");
+    goto out;
+  } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) {
+    connection_printf_to_buf(conn, "512 No auth clients specified\r\n");
+    goto out;
+  } else if ((auth_type == REND_BASIC_AUTH &&
+              smartlist_len(auth_clients) > 512) ||
+             (auth_type == REND_STEALTH_AUTH &&
+              smartlist_len(auth_clients) > 16)) {
+    connection_printf_to_buf(conn, "512 Too many auth clients\r\n");
+    goto out;
+  } else if (non_anonymous != rend_service_non_anonymous_mode_enabled(
+                                                              get_options())) {
+    /* If we failed, and the non-anonymous flag is set, Tor must be in
+     * anonymous hidden service mode.
+     * The error message changes based on the current Tor config:
+     * 512 Tor is in anonymous hidden service mode
+     * 512 Tor is in non-anonymous hidden service mode
+     * (I've deliberately written them out in full here to aid searchability.)
+     */
+    connection_printf_to_buf(conn, "512 Tor is in %sanonymous hidden service "
+                             "mode\r\n",
+                             non_anonymous ? "" : "non-");
+    goto out;
+  }
+
+  /* Parse the "keytype:keyblob" argument. */
+  int hs_version = 0;
+  add_onion_secret_key_t pk = { NULL };
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+  char *err_msg = NULL;
+
+  if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk,
+                              &key_new_alg, &key_new_blob, &pk, &hs_version,
+                              &err_msg) < 0) {
+    if (err_msg) {
+      connection_write_str_to_buf(err_msg, conn);
+      tor_free(err_msg);
+    }
+    goto out;
+  }
+  tor_assert(!err_msg);
+
+  /* Hidden service version 3 don't have client authentication support so if
+   * ClientAuth was given, send back an error. */
+  if (hs_version == HS_VERSION_THREE && auth_clients) {
+    connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n");
+    goto out;
+  }
+
+  /* Create the HS, using private key pk, client authentication auth_type,
+   * the list of auth_clients, and port config port_cfg.
+   * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
+   * regardless of success/failure.
+   */
+  char *service_id = NULL;
+  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
+                                         max_streams,
+                                         max_streams_close_circuit, auth_type,
+                                         auth_clients, &service_id);
+  port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
+  auth_clients = NULL; /* so is auth_clients */
+  switch (ret) {
+  case RSAE_OKAY:
+  {
+    if (detach) {
+      if (!detached_onion_services)
+        detached_onion_services = smartlist_new();
+      smartlist_add(detached_onion_services, service_id);
+    } else {
+      if (!conn->ephemeral_onion_services)
+        conn->ephemeral_onion_services = smartlist_new();
+      smartlist_add(conn->ephemeral_onion_services, service_id);
+    }
+
+    tor_assert(service_id);
+    connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id);
+    if (key_new_alg) {
+      tor_assert(key_new_blob);
+      connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n",
+                               key_new_alg, key_new_blob);
+    }
+    if (auth_created_clients) {
+      SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, {
+        char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie,
+                                                auth_type);
+        tor_assert(encoded);
+        connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n",
+                                 ac->client_name, encoded);
+        memwipe(encoded, 0, strlen(encoded));
+        tor_free(encoded);
+      });
+    }
+
+    connection_printf_to_buf(conn, "250 OK\r\n");
+    break;
+  }
+  case RSAE_BADPRIVKEY:
+    connection_printf_to_buf(conn, "551 Failed to generate onion address\r\n");
+    break;
+  case RSAE_ADDREXISTS:
+    connection_printf_to_buf(conn, "550 Onion address collision\r\n");
+    break;
+  case RSAE_BADVIRTPORT:
+    connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
+    break;
+  case RSAE_BADAUTH:
+    connection_printf_to_buf(conn, "512 Invalid client authorization\r\n");
+    break;
+  case RSAE_INTERNAL: /* FALLSTHROUGH */
+  default:
+    connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n");
+  }
+  if (key_new_blob) {
+    memwipe(key_new_blob, 0, strlen(key_new_blob));
+    tor_free(key_new_blob);
+  }
+
+ out:
+  if (port_cfgs) {
+    SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p,
+                      rend_service_port_config_free(p));
+    smartlist_free(port_cfgs);
+  }
+
+  if (auth_clients) {
+    SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac,
+                      rend_authorized_client_free(ac));
+    smartlist_free(auth_clients);
+  }
+  if (auth_created_clients) {
+    // Do not free entries; they are the same as auth_clients
+    smartlist_free(auth_created_clients);
+  }
+
+  SMARTLIST_FOREACH(args, char *, cp, {
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+  });
+  smartlist_free(args);
+  return 0;
+}
+
+/** Helper function to handle parsing the KeyType:KeyBlob argument to the
+ * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated
+ * and the private key not discarded, the algorithm and serialized private key,
+ * or NULL and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned key_new_blob and err_msg.
+ *
+ * Note: The error messages returned are deliberately vague to avoid echoing
+ * key material.
+ */
+STATIC int
+add_onion_helper_keyarg(const char *arg, int discard_pk,
+                        const char **key_new_alg_out, char **key_new_blob_out,
+                        add_onion_secret_key_t *decoded_key, int *hs_version,
+                        char **err_msg_out)
+{
+  smartlist_t *key_args = smartlist_new();
+  crypto_pk_t *pk = NULL;
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+  char *err_msg = NULL;
+  int ret = -1;
+
+  smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
+  if (smartlist_len(key_args) != 2) {
+    err_msg = tor_strdup("512 Invalid key type/blob\r\n");
+    goto err;
+  }
+
+  /* The format is "KeyType:KeyBlob". */
+  static const char *key_type_new = "NEW";
+  static const char *key_type_best = "BEST";
+  static const char *key_type_rsa1024 = "RSA1024";
+  static const char *key_type_ed25519_v3 = "ED25519-V3";
+
+  const char *key_type = smartlist_get(key_args, 0);
+  const char *key_blob = smartlist_get(key_args, 1);
+
+  if (!strcasecmp(key_type_rsa1024, key_type)) {
+    /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
+    pk = crypto_pk_base64_decode(key_blob, strlen(key_blob));
+    if (!pk) {
+      err_msg = tor_strdup("512 Failed to decode RSA key\r\n");
+      goto err;
+    }
+    if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
+      crypto_pk_free(pk);
+      err_msg = tor_strdup("512 Invalid RSA key size\r\n");
+      goto err;
+    }
+    decoded_key->v2 = pk;
+    *hs_version = HS_VERSION_TWO;
+  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
+    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
+    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
+                      strlen(key_blob)) != sizeof(sk->seckey)) {
+      tor_free(sk);
+      err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n");
+      goto err;
+    }
+    decoded_key->v3 = sk;
+    *hs_version = HS_VERSION_THREE;
+  } else if (!strcasecmp(key_type_new, key_type)) {
+    /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
+    if (!strcasecmp(key_type_rsa1024, key_blob) ||
+        !strcasecmp(key_type_best, key_blob)) {
+      /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */
+      pk = crypto_pk_new();
+      if (crypto_pk_generate_key(pk)) {
+        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
+                     key_type_rsa1024);
+        goto err;
+      }
+      if (!discard_pk) {
+        if (crypto_pk_base64_encode(pk, &key_new_blob)) {
+          crypto_pk_free(pk);
+          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
+                       key_type_rsa1024);
+          goto err;
+        }
+        key_new_alg = key_type_rsa1024;
+      }
+      decoded_key->v2 = pk;
+      *hs_version = HS_VERSION_TWO;
+    } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) {
+      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+      if (ed25519_secret_key_generate(sk, 1) < 0) {
+        tor_free(sk);
+        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
+                     key_type_ed25519_v3);
+        goto err;
+      }
+      if (!discard_pk) {
+        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
+        key_new_blob = tor_malloc_zero(len);
+        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
+                          sizeof(sk->seckey), 0) != (len - 1)) {
+          tor_free(sk);
+          tor_free(key_new_blob);
+          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
+                       key_type_ed25519_v3);
+          goto err;
+        }
+        key_new_alg = key_type_ed25519_v3;
+      }
+      decoded_key->v3 = sk;
+      *hs_version = HS_VERSION_THREE;
+    } else {
+      err_msg = tor_strdup("513 Invalid key type\r\n");
+      goto err;
+    }
+  } else {
+    err_msg = tor_strdup("513 Invalid key type\r\n");
+    goto err;
+  }
+
+  /* Succeeded in loading or generating a private key. */
+  ret = 0;
+
+ err:
+  SMARTLIST_FOREACH(key_args, char *, cp, {
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+  });
+  smartlist_free(key_args);
+
+  if (err_msg_out) {
+    *err_msg_out = err_msg;
+  } else {
+    tor_free(err_msg);
+  }
+  *key_new_alg_out = key_new_alg;
+  *key_new_blob_out = key_new_blob;
+
+  return ret;
+}
+
+/** Helper function to handle parsing a ClientAuth argument to the
+ * ADD_ONION command.  Return a new rend_authorized_client_t, or NULL
+ * and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned auth_client and err_msg.
+ *
+ * If 'created' is specified, it will be set to 1 when a new cookie has
+ * been generated.
+ */
+STATIC rend_authorized_client_t *
+add_onion_helper_clientauth(const char *arg, int *created, char **err_msg)
+{
+  int ok = 0;
+
+  tor_assert(arg);
+  tor_assert(created);
+  tor_assert(err_msg);
+  *err_msg = NULL;
+
+  smartlist_t *auth_args = smartlist_new();
+  rend_authorized_client_t *client =
+    tor_malloc_zero(sizeof(rend_authorized_client_t));
+  smartlist_split_string(auth_args, arg, ":", 0, 0);
+  if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) {
+    *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n");
+    goto err;
+  }
+  client->client_name = tor_strdup(smartlist_get(auth_args, 0));
+  if (smartlist_len(auth_args) == 2) {
+    char *decode_err_msg = NULL;
+    if (rend_auth_decode_cookie(smartlist_get(auth_args, 1),
+                                client->descriptor_cookie,
+                                NULL, &decode_err_msg) < 0) {
+      tor_assert(decode_err_msg);
+      tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg);
+      tor_free(decode_err_msg);
+      goto err;
+    }
+    *created = 0;
+  } else {
+    crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
+    *created = 1;
+  }
+
+  if (!rend_valid_client_name(client->client_name)) {
+    *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n");
+    goto err;
+  }
+
+  ok = 1;
+ err:
+  SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item));
+  smartlist_free(auth_args);
+  if (!ok) {
+    rend_authorized_client_free(client);
+    client = NULL;
+  }
+  return client;
+}
+
+/** Called when we get a DEL_ONION command; parse the body, and remove
+ * the existing ephemeral Onion Service. */
+static int
+handle_control_del_onion(control_connection_t *conn,
+                          uint32_t len,
+                          const char *body)
+{
+  int hs_version = 0;
+  smartlist_t *args;
+  (void) len; /* body is nul-terminated; it's safe to ignore the length */
+  args = getargs_helper("DEL_ONION", conn, body, 1, 1);
+  if (!args)
+    return 0;
+
+  const char *service_id = smartlist_get(args, 0);
+  if (rend_valid_v2_service_id(service_id)) {
+    hs_version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(service_id)) {
+    hs_version = HS_VERSION_THREE;
+  } else {
+    connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n");
+    goto out;
+  }
+
+  /* Determine if the onion service belongs to this particular control
+   * connection, or if it is in the global list of detached services.  If it
+   * is in neither, either the service ID is invalid in some way, or it
+   * explicitly belongs to a different control connection, and an error
+   * should be returned.
+   */
+  smartlist_t *services[2] = {
+    conn->ephemeral_onion_services,
+    detached_onion_services
+  };
+  smartlist_t *onion_services = NULL;
+  int idx = -1;
+  for (size_t i = 0; i < ARRAY_LENGTH(services); i++) {
+    idx = smartlist_string_pos(services[i], service_id);
+    if (idx != -1) {
+      onion_services = services[i];
+      break;
+    }
+  }
+  if (onion_services == NULL) {
+    connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n");
+  } else {
+    int ret = -1;
+    switch (hs_version) {
+    case HS_VERSION_TWO:
+      ret = rend_service_del_ephemeral(service_id);
+      break;
+    case HS_VERSION_THREE:
+      ret = hs_service_del_ephemeral(service_id);
+      break;
+    default:
+      /* The ret value will be -1 thus hitting the warning below. This should
+       * never happen because of the check at the start of the function. */
+      break;
+    }
+    if (ret < 0) {
+      /* This should *NEVER* fail, since the service is on either the
+       * per-control connection list, or the global one.
+       */
+      log_warn(LD_BUG, "Failed to remove Onion Service %s.",
+               escaped(service_id));
+      tor_fragile_assert();
+    }
+
+    /* Remove/scrub the service_id from the appropriate list. */
+    char *cp = smartlist_get(onion_services, idx);
+    smartlist_del(onion_services, idx);
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+
+    send_control_done(conn);
+  }
+
+ out:
+  SMARTLIST_FOREACH(args, char *, cp, {
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+  });
+  smartlist_free(args);
+  return 0;
+}
+
+/** Called when <b>conn</b> has no more bytes left on its outbuf. */
+int
+connection_control_finished_flushing(control_connection_t *conn)
+{
+  tor_assert(conn);
+  return 0;
+}
+
+/** Called when <b>conn</b> has gotten its socket closed. */
+int
+connection_control_reached_eof(control_connection_t *conn)
+{
+  tor_assert(conn);
+
+  log_info(LD_CONTROL,"Control connection reached EOF. Closing.");
+  connection_mark_for_close(TO_CONN(conn));
+  return 0;
+}
+
+/** Shut down this Tor instance in the same way that SIGINT would, but
+ * with a log message appropriate for the loss of an owning controller. */
+static void
+lost_owning_controller(const char *owner_type, const char *loss_manner)
+{
+  log_notice(LD_CONTROL, "Owning controller %s has %s -- exiting now.",
+             owner_type, loss_manner);
+
+  activate_signal(SIGTERM);
+}
+
+/** Called when <b>conn</b> is being freed. */
+void
+connection_control_closed(control_connection_t *conn)
+{
+  tor_assert(conn);
+
+  conn->event_mask = 0;
+  control_update_global_event_mask();
+
+  /* Close all ephemeral Onion Services if any.
+   * The list and it's contents are scrubbed/freed in connection_free_.
+   */
+  if (conn->ephemeral_onion_services) {
+    SMARTLIST_FOREACH_BEGIN(conn->ephemeral_onion_services, char *, cp) {
+      if (rend_valid_v2_service_id(cp)) {
+        rend_service_del_ephemeral(cp);
+      } else if (hs_address_is_valid(cp)) {
+        hs_service_del_ephemeral(cp);
+      } else {
+        /* An invalid .onion in our list should NEVER happen */
+        tor_fragile_assert();
+      }
+    } SMARTLIST_FOREACH_END(cp);
+  }
+
+  if (conn->is_owning_control_connection) {
+    lost_owning_controller("connection", "closed");
+  }
+}
+
+/** Return true iff <b>cmd</b> is allowable (or at least forgivable) at this
+ * stage of the protocol. */
+static int
+is_valid_initial_command(control_connection_t *conn, const char *cmd)
+{
+  if (conn->base_.state == CONTROL_CONN_STATE_OPEN)
+    return 1;
+  if (!strcasecmp(cmd, "PROTOCOLINFO"))
+    return (!conn->have_sent_protocolinfo &&
+            conn->safecookie_client_hash == NULL);
+  if (!strcasecmp(cmd, "AUTHCHALLENGE"))
+    return (conn->safecookie_client_hash == NULL);
+  if (!strcasecmp(cmd, "AUTHENTICATE") ||
+      !strcasecmp(cmd, "QUIT"))
+    return 1;
+  return 0;
+}
+
+/** Do not accept any control command of more than 1MB in length.  Anything
+ * that needs to be anywhere near this long probably means that one of our
+ * interfaces is broken. */
+#define MAX_COMMAND_LINE_LENGTH (1024*1024)
+
+/** Wrapper around peek_buf_has_control0 command: presents the same
+ * interface as that underlying functions, but takes a connection_t intead of
+ * a buf_t.
+ */
+static int
+peek_connection_has_control0_command(connection_t *conn)
+{
+  return peek_buf_has_control0_command(conn->inbuf);
+}
+
+static int
+peek_connection_has_http_command(connection_t *conn)
+{
+  return peek_buf_has_http_command(conn->inbuf);
+}
+
+static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 501 Tor ControlPort is not an HTTP proxy"
+  "\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>Tor's ControlPort is not an HTTP proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>Tor's ControlPort is not an HTTP proxy</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use Tor's control port"
+  " as an HTTP proxy.\n"
+  "This is not correct: Tor's default SOCKS proxy port is 9050.\n"
+  "Please configure your client accordingly.\n"
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "<!-- Plus this comment, to make the body response more than 512 bytes, so "
+  "     IE will be willing to display it. Comment comment comment comment "
+  "     comment comment comment comment comment comment comment comment.-->\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
+/** Called when data has arrived on a v1 control connection: Try to fetch
+ * commands from conn->inbuf, and execute them.
+ */
+int
+connection_control_process_inbuf(control_connection_t *conn)
+{
+  size_t data_len;
+  uint32_t cmd_data_len;
+  int cmd_len;
+  char *args;
+
+  tor_assert(conn);
+  tor_assert(conn->base_.state == CONTROL_CONN_STATE_OPEN ||
+             conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH);
+
+  if (!conn->incoming_cmd) {
+    conn->incoming_cmd = tor_malloc(1024);
+    conn->incoming_cmd_len = 1024;
+    conn->incoming_cmd_cur_len = 0;
+  }
+
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_control0_command(TO_CONN(conn))) {
+    /* Detect v0 commands and send a "no more v0" message. */
+    size_t body_len;
+    char buf[128];
+    set_uint16(buf+2, htons(0x0000)); /* type == error */
+    set_uint16(buf+4, htons(0x0001)); /* code == internal error */
+    strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 "
+            "and later; upgrade your controller.",
+            sizeof(buf)-6);
+    body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
+    set_uint16(buf+0, htons(body_len));
+    connection_buf_add(buf, 4+body_len, TO_CONN(conn));
+
+    connection_mark_and_flush(TO_CONN(conn));
+    return 0;
+  }
+
+  /* If the user has the HTTP proxy port and the control port confused. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_http_command(TO_CONN(conn))) {
+    connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
+    log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
+    connection_mark_and_flush(TO_CONN(conn));
+    return 0;
+  }
+
+ again:
+  while (1) {
+    size_t last_idx;
+    int r;
+    /* First, fetch a line. */
+    do {
+      data_len = conn->incoming_cmd_len - conn->incoming_cmd_cur_len;
+      r = connection_buf_get_line(TO_CONN(conn),
+                              conn->incoming_cmd+conn->incoming_cmd_cur_len,
+                              &data_len);
+      if (r == 0)
+        /* Line not all here yet. Wait. */
+        return 0;
+      else if (r == -1) {
+        if (data_len + conn->incoming_cmd_cur_len > MAX_COMMAND_LINE_LENGTH) {
+          connection_write_str_to_buf("500 Line too long.\r\n", conn);
+          connection_stop_reading(TO_CONN(conn));
+          connection_mark_and_flush(TO_CONN(conn));
+        }
+        while (conn->incoming_cmd_len < data_len+conn->incoming_cmd_cur_len)
+          conn->incoming_cmd_len *= 2;
+        conn->incoming_cmd = tor_realloc(conn->incoming_cmd,
+                                         conn->incoming_cmd_len);
+      }
+    } while (r != 1);
+
+    tor_assert(data_len);
+
+    last_idx = conn->incoming_cmd_cur_len;
+    conn->incoming_cmd_cur_len += (int)data_len;
+
+    /* We have appended a line to incoming_cmd.  Is the command done? */
+    if (last_idx == 0 && *conn->incoming_cmd != '+')
+      /* One line command, didn't start with '+'. */
+      break;
+    /* XXXX this code duplication is kind of dumb. */
+    if (last_idx+3 == conn->incoming_cmd_cur_len &&
+        tor_memeq(conn->incoming_cmd + last_idx, ".\r\n", 3)) {
+      /* Just appended ".\r\n"; we're done. Remove it. */
+      conn->incoming_cmd[last_idx] = '\0';
+      conn->incoming_cmd_cur_len -= 3;
+      break;
+    } else if (last_idx+2 == conn->incoming_cmd_cur_len &&
+               tor_memeq(conn->incoming_cmd + last_idx, ".\n", 2)) {
+      /* Just appended ".\n"; we're done. Remove it. */
+      conn->incoming_cmd[last_idx] = '\0';
+      conn->incoming_cmd_cur_len -= 2;
+      break;
+    }
+    /* Otherwise, read another line. */
+  }
+  data_len = conn->incoming_cmd_cur_len;
+  /* Okay, we now have a command sitting on conn->incoming_cmd. See if we
+   * recognize it.
+   */
+  cmd_len = 0;
+  while ((size_t)cmd_len < data_len
+         && !TOR_ISSPACE(conn->incoming_cmd[cmd_len]))
+    ++cmd_len;
+
+  conn->incoming_cmd[cmd_len]='\0';
+  args = conn->incoming_cmd+cmd_len+1;
+  tor_assert(data_len>(size_t)cmd_len);
+  data_len -= (cmd_len+1); /* skip the command and NUL we added after it */
+  while (TOR_ISSPACE(*args)) {
+    ++args;
+    --data_len;
+  }
+
+  /* If the connection is already closing, ignore further commands */
+  if (TO_CONN(conn)->marked_for_close) {
+    return 0;
+  }
+
+  /* Otherwise, Quit is always valid. */
+  if (!strcasecmp(conn->incoming_cmd, "QUIT")) {
+    connection_write_str_to_buf("250 closing connection\r\n", conn);
+    connection_mark_and_flush(TO_CONN(conn));
+    return 0;
+  }
+
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      !is_valid_initial_command(conn, conn->incoming_cmd)) {
+    connection_write_str_to_buf("514 Authentication required.\r\n", conn);
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  }
+
+  if (data_len >= UINT32_MAX) {
+    connection_write_str_to_buf("500 A 4GB command? Nice try.\r\n", conn);
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  }
+
+  /* XXXX Why is this not implemented as a table like the GETINFO
+   * items are?  Even handling the plus signs at the beginnings of
+   * commands wouldn't be very hard with proper macros. */
+  cmd_data_len = (uint32_t)data_len;
+  if (!strcasecmp(conn->incoming_cmd, "SETCONF")) {
+    if (handle_control_setconf(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "RESETCONF")) {
+    if (handle_control_resetconf(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "GETCONF")) {
+    if (handle_control_getconf(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "+LOADCONF")) {
+    if (handle_control_loadconf(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "SETEVENTS")) {
+    if (handle_control_setevents(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "AUTHENTICATE")) {
+    if (handle_control_authenticate(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "SAVECONF")) {
+    if (handle_control_saveconf(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "SIGNAL")) {
+    if (handle_control_signal(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) {
+    if (handle_control_takeownership(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) {
+    if (handle_control_mapaddress(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "GETINFO")) {
+    if (handle_control_getinfo(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "EXTENDCIRCUIT")) {
+    if (handle_control_extendcircuit(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "SETCIRCUITPURPOSE")) {
+    if (handle_control_setcircuitpurpose(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "SETROUTERPURPOSE")) {
+    connection_write_str_to_buf("511 SETROUTERPURPOSE is obsolete.\r\n", conn);
+  } else if (!strcasecmp(conn->incoming_cmd, "ATTACHSTREAM")) {
+    if (handle_control_attachstream(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "+POSTDESCRIPTOR")) {
+    if (handle_control_postdescriptor(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "REDIRECTSTREAM")) {
+    if (handle_control_redirectstream(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "CLOSESTREAM")) {
+    if (handle_control_closestream(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "CLOSECIRCUIT")) {
+    if (handle_control_closecircuit(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "USEFEATURE")) {
+    if (handle_control_usefeature(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "RESOLVE")) {
+    if (handle_control_resolve(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "PROTOCOLINFO")) {
+    if (handle_control_protocolinfo(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) {
+    if (handle_control_authchallenge(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) {
+    if (handle_control_dropguards(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "HSFETCH")) {
+    if (handle_control_hsfetch(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "+HSPOST")) {
+    if (handle_control_hspost(conn, cmd_data_len, args))
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "ADD_ONION")) {
+    int ret = handle_control_add_onion(conn, cmd_data_len, args);
+    memwipe(args, 0, cmd_data_len); /* Scrub the private key. */
+    if (ret)
+      return -1;
+  } else if (!strcasecmp(conn->incoming_cmd, "DEL_ONION")) {
+    int ret = handle_control_del_onion(conn, cmd_data_len, args);
+    memwipe(args, 0, cmd_data_len); /* Scrub the service id/pk. */
+    if (ret)
+      return -1;
+  } else {
+    connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n",
+                             conn->incoming_cmd);
+  }
+
+  conn->incoming_cmd_cur_len = 0;
+  goto again;
+}
+
+/** Something major has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+int
+control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                             int reason_code)
+{
+  const char *status;
+  char reasons[64] = "";
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS))
+    return 0;
+  tor_assert(circ);
+
+  switch (tp)
+    {
+    case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case CIRC_EVENT_BUILT: status = "BUILT"; break;
+    case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break;
+    case CIRC_EVENT_FAILED: status = "FAILED"; break;
+    case CIRC_EVENT_CLOSED: status = "CLOSED"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) {
+    const char *reason_str = circuit_end_reason_to_control_string(reason_code);
+    char unk_reason_buf[16];
+    if (!reason_str) {
+      tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code);
+      reason_str = unk_reason_buf;
+    }
+    if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=DESTROYED REMOTE_REASON=%s", reason_str);
+    } else {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=%s", reason_str);
+    }
+  }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS,
+                                "650 CIRC %lu %s%s%s%s\r\n",
+                                (unsigned long)circ->global_identifier,
+                                status, sp,
+                                circdesc,
+                                reasons);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/** Something minor has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+static int
+control_event_circuit_status_minor(origin_circuit_t *circ,
+                                   circuit_status_minor_event_t e,
+                                   int purpose, const struct timeval *tv)
+{
+  const char *event_desc;
+  char event_tail[160] = "";
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR))
+    return 0;
+  tor_assert(circ);
+
+  switch (e)
+    {
+    case CIRC_MINOR_EVENT_PURPOSE_CHANGED:
+      event_desc = "PURPOSE_CHANGED";
+
+      {
+        /* event_tail can currently be up to 68 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "");
+      }
+
+      break;
+    case CIRC_MINOR_EVENT_CANNIBALIZED:
+      event_desc = "CANNIBALIZED";
+
+      {
+        /* event_tail can currently be up to 130 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        const struct timeval *old_timestamp_began = tv;
+        char tbuf[ISO_TIME_USEC_LEN+1];
+        format_iso_time_nospace_usec(tbuf, old_timestamp_began);
+
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "",
+                     tbuf);
+      }
+
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)e);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS_MINOR,
+                       "650 CIRC_MINOR %lu %s%s%s%s\r\n",
+                       (unsigned long)circ->global_identifier,
+                       event_desc, sp,
+                       circdesc,
+                       event_tail);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>: tell any
+ * interested controllers.
+ */
+int
+control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                      int old_purpose)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+                                            old_purpose,
+                                            NULL);
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>, and its
+ * created-time from <b>old_tv_created</b>: tell any interested controllers.
+ */
+int
+control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                   int old_purpose,
+                                   const struct timeval *old_tv_created)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_CANNIBALIZED,
+                                            old_purpose,
+                                            old_tv_created);
+}
+
+/** Given an AP connection <b>conn</b> and a <b>len</b>-character buffer
+ * <b>buf</b>, determine the address:port combination requested on
+ * <b>conn</b>, and write it to <b>buf</b>.  Return 0 on success, -1 on
+ * failure. */
+static int
+write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len)
+{
+  char buf2[256];
+  if (conn->chosen_exit_name)
+    if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0)
+      return -1;
+  if (!conn->socks_request)
+    return -1;
+  if (tor_snprintf(buf, len, "%s%s%s:%d",
+               conn->socks_request->address,
+               conn->chosen_exit_name ? buf2 : "",
+               !conn->chosen_exit_name && connection_edge_is_rendezvous_stream(
+                                     ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "",
+               conn->socks_request->port)<0)
+    return -1;
+  return 0;
+}
+
+/** Something has happened to the stream associated with AP connection
+ * <b>conn</b>: tell any interested control connections. */
+int
+control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
+                            int reason_code)
+{
+  char reason_buf[64];
+  char addrport_buf[64];
+  const char *status;
+  circuit_t *circ;
+  origin_circuit_t *origin_circ = NULL;
+  char buf[256];
+  const char *purpose = "";
+  tor_assert(conn->socks_request);
+
+  if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS))
+    return 0;
+
+  if (tp == STREAM_EVENT_CLOSED &&
+      (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED))
+    return 0;
+
+  write_stream_target_to_buf(conn, buf, sizeof(buf));
+
+  reason_buf[0] = '\0';
+  switch (tp)
+    {
+    case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break;
+    case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break;
+    case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break;
+    case STREAM_EVENT_FAILED: status = "FAILED"; break;
+    case STREAM_EVENT_CLOSED: status = "CLOSED"; break;
+    case STREAM_EVENT_NEW: status = "NEW"; break;
+    case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break;
+    case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break;
+    case STREAM_EVENT_REMAP: status = "REMAP"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (reason_code && (tp == STREAM_EVENT_FAILED ||
+                      tp == STREAM_EVENT_CLOSED ||
+                      tp == STREAM_EVENT_FAILED_RETRIABLE)) {
+    const char *reason_str = stream_end_reason_to_control_string(reason_code);
+    char *r = NULL;
+    if (!reason_str) {
+      tor_asprintf(&r, " UNKNOWN_%d", reason_code);
+      reason_str = r;
+    }
+    if (reason_code & END_STREAM_REASON_FLAG_REMOTE)
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=END REMOTE_REASON=%s", reason_str);
+    else
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=%s", reason_str);
+    tor_free(r);
+  } else if (reason_code && tp == STREAM_EVENT_REMAP) {
+    switch (reason_code) {
+    case REMAP_STREAM_SOURCE_CACHE:
+      strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf));
+      break;
+    case REMAP_STREAM_SOURCE_EXIT:
+      strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf));
+      break;
+    default:
+      tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d",
+                   reason_code);
+      /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */
+      break;
+    }
+  }
+
+  if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) {
+    /*
+     * When the control conn is an AF_UNIX socket and we have no address,
+     * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in
+     * dnsserv.c.
+     */
+    if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) {
+      tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d",
+                   ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port);
+    } else {
+      /*
+       * else leave it blank so control on AF_UNIX doesn't need to make
+       * something up.
+       */
+      addrport_buf[0] = '\0';
+    }
+  } else {
+    addrport_buf[0] = '\0';
+  }
+
+  if (tp == STREAM_EVENT_NEW_RESOLVE) {
+    purpose = " PURPOSE=DNS_REQUEST";
+  } else if (tp == STREAM_EVENT_NEW) {
+    if (conn->use_begindir) {
+      connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn;
+      int linked_dir_purpose = -1;
+      if (linked && linked->type == CONN_TYPE_DIR)
+        linked_dir_purpose = linked->purpose;
+      if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose))
+        purpose = " PURPOSE=DIR_UPLOAD";
+      else
+        purpose = " PURPOSE=DIR_FETCH";
+    } else
+      purpose = " PURPOSE=USER";
+  }
+
+  circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+  if (circ && CIRCUIT_IS_ORIGIN(circ))
+    origin_circ = TO_ORIGIN_CIRCUIT(circ);
+  send_control_event(EVENT_STREAM_STATUS,
+                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n",
+                     (ENTRY_TO_CONN(conn)->global_identifier),
+                     status,
+                        origin_circ?
+                           (unsigned long)origin_circ->global_identifier : 0ul,
+                        buf, reason_buf, addrport_buf, purpose);
+
+  /* XXX need to specify its intended exit, etc? */
+
+  return 0;
+}
+
+/** Figure out the best name for the target router of an OR connection
+ * <b>conn</b>, and write it into the <b>len</b>-character buffer
+ * <b>name</b>. */
+static void
+orconn_target_get_name(char *name, size_t len, or_connection_t *conn)
+{
+  const node_t *node = node_get_by_id(conn->identity_digest);
+  if (node) {
+    tor_assert(len > MAX_VERBOSE_NICKNAME_LEN);
+    node_get_verbose_nickname(node, name);
+  } else if (! tor_digest_is_zero(conn->identity_digest)) {
+    name[0] = '$';
+    base16_encode(name+1, len-1, conn->identity_digest,
+                  DIGEST_LEN);
+  } else {
+    tor_snprintf(name, len, "%s:%d",
+                 conn->base_.address, conn->base_.port);
+  }
+}
+
+/** Called when the status of an OR connection <b>conn</b> changes: tell any
+ * interested control connections. <b>tp</b> is the new status for the
+ * connection.  If <b>conn</b> has just closed or failed, then <b>reason</b>
+ * may be the reason why.
+ */
+int
+control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
+                             int reason)
+{
+  int ncircs = 0;
+  const char *status;
+  char name[128];
+  char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */
+
+  if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS))
+    return 0;
+
+  switch (tp)
+    {
+    case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break;
+    case OR_CONN_EVENT_FAILED: status = "FAILED"; break;
+    case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break;
+    case OR_CONN_EVENT_NEW: status = "NEW"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (conn->chan) {
+    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
+  } else {
+    ncircs = 0;
+  }
+  ncircs += connection_or_get_num_circuits(conn);
+  if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
+    tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs);
+  }
+
+  orconn_target_get_name(name, sizeof(name), conn);
+  send_control_event(EVENT_OR_CONN_STATUS,
+                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
+                              name, status,
+                              reason ? " REASON=" : "",
+                              orconn_end_reason_to_control_string(reason),
+                              ncircs_buf,
+                              (conn->base_.global_identifier));
+
+  return 0;
+}
+
+/**
+ * Print out STREAM_BW event for a single conn
+ */
+int
+control_event_stream_bandwidth(edge_connection_t *edge_conn)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    if (!edge_conn->n_read && !edge_conn->n_written)
+      return 0;
+
+    tor_gettimeofday(&now);
+    format_iso_time_nospace_usec(tbuf, &now);
+    send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                       (edge_conn->base_.global_identifier),
+                       (unsigned long)edge_conn->n_read,
+                       (unsigned long)edge_conn->n_written,
+                       tbuf);
+
+    edge_conn->n_written = edge_conn->n_read = 0;
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth streams have used. */
+int
+control_event_stream_bandwidth_used(void)
+{
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    smartlist_t *conns = get_connection_array();
+    edge_connection_t *edge_conn;
+    struct timeval now;
+    char tbuf[ISO_TIME_USEC_LEN+1];
+
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
+    {
+        if (conn->type != CONN_TYPE_AP)
+          continue;
+        edge_conn = TO_EDGE_CONN(conn);
+        if (!edge_conn->n_read && !edge_conn->n_written)
+          continue;
+
+        tor_gettimeofday(&now);
+        format_iso_time_nospace_usec(tbuf, &now);
+        send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                           (edge_conn->base_.global_identifier),
+                           (unsigned long)edge_conn->n_read,
+                           (unsigned long)edge_conn->n_written,
+                           tbuf);
+
+        edge_conn->n_written = edge_conn->n_read = 0;
+    }
+    SMARTLIST_FOREACH_END(conn);
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control connections
+ * how much bandwidth origin circuits have used. */
+int
+control_event_circ_bandwidth_used(void)
+{
+  origin_circuit_t *ocirc;
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
+    if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
+      continue;
+    tor_gettimeofday(&now);
+    format_iso_time_nospace_usec(tbuf, &now);
+    send_control_event(EVENT_CIRC_BANDWIDTH_USED,
+                       "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
+                       "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
+                       "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
+                       ocirc->global_identifier,
+                       (unsigned long)ocirc->n_read_circ_bw,
+                       (unsigned long)ocirc->n_written_circ_bw,
+                       tbuf,
+                       (unsigned long)ocirc->n_delivered_read_circ_bw,
+                       (unsigned long)ocirc->n_overhead_read_circ_bw,
+                       (unsigned long)ocirc->n_delivered_written_circ_bw,
+                       (unsigned long)ocirc->n_overhead_written_circ_bw);
+    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+  }
+  SMARTLIST_FOREACH_END(circ);
+
+  return 0;
+}
+
+/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
+  * bandwidth counters. */
+int
+control_event_conn_bandwidth(connection_t *conn)
+{
+  const char *conn_type_str;
+  if (!get_options()->TestingEnableConnBwEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CONN_BW))
+    return 0;
+  if (!conn->n_read_conn_bw && !conn->n_written_conn_bw)
+    return 0;
+  switch (conn->type) {
+    case CONN_TYPE_OR:
+      conn_type_str = "OR";
+      break;
+    case CONN_TYPE_DIR:
+      conn_type_str = "DIR";
+      break;
+    case CONN_TYPE_EXIT:
+      conn_type_str = "EXIT";
+      break;
+    default:
+      return 0;
+  }
+  send_control_event(EVENT_CONN_BW,
+                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
+                     "READ=%lu WRITTEN=%lu\r\n",
+                     (conn->global_identifier),
+                     conn_type_str,
+                     (unsigned long)conn->n_read_conn_bw,
+                     (unsigned long)conn->n_written_conn_bw);
+  conn->n_written_conn_bw = conn->n_read_conn_bw = 0;
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth connections have used. */
+int
+control_event_conn_bandwidth_used(void)
+{
+  if (get_options()->TestingEnableConnBwEvent &&
+      EVENT_IS_INTERESTING(EVENT_CONN_BW)) {
+    SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn,
+                      control_event_conn_bandwidth(conn));
+  }
+  return 0;
+}
+
+/** Helper: iterate over cell statistics of <b>circ</b> and sum up added
+ * cells, removed cells, and waiting times by cell command and direction.
+ * Store results in <b>cell_stats</b>.  Free cell statistics of the
+ * circuit afterwards. */
+void
+sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats)
+{
+  memset(cell_stats, 0, sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats,
+                          const testing_cell_stats_entry_t *, ent) {
+    tor_assert(ent->command <= CELL_COMMAND_MAX_);
+    if (!ent->removed && !ent->exitward) {
+      cell_stats->added_cells_appward[ent->command] += 1;
+    } else if (!ent->removed && ent->exitward) {
+      cell_stats->added_cells_exitward[ent->command] += 1;
+    } else if (!ent->exitward) {
+      cell_stats->removed_cells_appward[ent->command] += 1;
+      cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10;
+    } else {
+      cell_stats->removed_cells_exitward[ent->command] += 1;
+      cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10;
+    }
+  } SMARTLIST_FOREACH_END(ent);
+  circuit_clear_testing_cell_stats(circ);
+}
+
+/** Helper: append a cell statistics string to <code>event_parts</code>,
+ * prefixed with <code>key</code>=.  Statistics consist of comma-separated
+ * key:value pairs with lower-case command strings as keys and cell
+ * numbers or total waiting times as values.  A key:value pair is included
+ * if the entry in <code>include_if_non_zero</code> is not zero, but with
+ * the (possibly zero) entry from <code>number_to_include</code>.  Both
+ * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1.  If no
+ * entry in <code>include_if_non_zero</code> is positive, no string will
+ * be added to <code>event_parts</code>. */
+void
+append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
+                             const uint64_t *include_if_non_zero,
+                             const uint64_t *number_to_include)
+{
+  smartlist_t *key_value_strings = smartlist_new();
+  int i;
+  for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
+    if (include_if_non_zero[i] > 0) {
+      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
+                             cell_command_to_string(i),
+                             (number_to_include[i]));
+    }
+  }
+  if (smartlist_len(key_value_strings) > 0) {
+    char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL);
+    smartlist_add_asprintf(event_parts, "%s=%s", key, joined);
+    SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp));
+    tor_free(joined);
+  }
+  smartlist_free(key_value_strings);
+}
+
+/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a
+ * CELL_STATS event and write result string to <b>event_string</b>. */
+void
+format_cell_stats(char **event_string, circuit_t *circ,
+                  cell_stats_t *cell_stats)
+{
+  smartlist_t *event_parts = smartlist_new();
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "ID=%lu",
+                 (unsigned long)ocirc->global_identifier);
+  } else if (TO_OR_CIRCUIT(circ)->p_chan) {
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
+                 (unsigned long)or_circ->p_circ_id);
+    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
+                 (or_circ->p_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "InboundAdded",
+                                 cell_stats->added_cells_appward,
+                                 cell_stats->added_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundRemoved",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->removed_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundTime",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->total_time_appward);
+  }
+  if (circ->n_chan) {
+    smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
+                     (unsigned long)circ->n_circ_id);
+    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
+                 (circ->n_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "OutboundAdded",
+                                 cell_stats->added_cells_exitward,
+                                 cell_stats->added_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundRemoved",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->removed_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundTime",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->total_time_exitward);
+  }
+  *event_string = smartlist_join_strings(event_parts, " ", 0, NULL);
+  SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp));
+  smartlist_free(event_parts);
+}
+
+/** A second or more has elapsed: tell any interested control connection
+ * how many cells have been processed for a given circuit. */
+int
+control_event_circuit_cell_stats(void)
+{
+  cell_stats_t *cell_stats;
+  char *event_string;
+  if (!get_options()->TestingEnableCellStatsEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
+    return 0;
+  cell_stats = tor_malloc(sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!circ->testing_cell_stats)
+      continue;
+    sum_up_cell_stats_by_command(circ, cell_stats);
+    format_cell_stats(&event_string, circ, cell_stats);
+    send_control_event(EVENT_CELL_STATS,
+                       "650 CELL_STATS %s\r\n", event_string);
+    tor_free(event_string);
+  }
+  SMARTLIST_FOREACH_END(circ);
+  tor_free(cell_stats);
+  return 0;
+}
+
+/* about 5 minutes worth. */
+#define N_BW_EVENTS_TO_CACHE 300
+/* Index into cached_bw_events to next write. */
+static int next_measurement_idx = 0;
+/* number of entries set in n_measurements */
+static int n_measurements = 0;
+static struct cached_bw_event_s {
+  uint32_t n_read;
+  uint32_t n_written;
+} cached_bw_events[N_BW_EVENTS_TO_CACHE];
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth we used. */
+int
+control_event_bandwidth_used(uint32_t n_read, uint32_t n_written)
+{
+  cached_bw_events[next_measurement_idx].n_read = n_read;
+  cached_bw_events[next_measurement_idx].n_written = n_written;
+  if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE)
+    next_measurement_idx = 0;
+  if (n_measurements < N_BW_EVENTS_TO_CACHE)
+    ++n_measurements;
+
+  if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) {
+    send_control_event(EVENT_BANDWIDTH_USED,
+                       "650 BW %lu %lu\r\n",
+                       (unsigned long)n_read,
+                       (unsigned long)n_written);
+  }
+
+  return 0;
+}
+
+STATIC char *
+get_bw_samples(void)
+{
+  int i;
+  int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements)
+    % N_BW_EVENTS_TO_CACHE;
+  tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+
+  smartlist_t *elements = smartlist_new();
+
+  for (i = 0; i < n_measurements; ++i) {
+    tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+    const struct cached_bw_event_s *bwe = &cached_bw_events[idx];
+
+    smartlist_add_asprintf(elements, "%u,%u",
+                           (unsigned)bwe->n_read,
+                           (unsigned)bwe->n_written);
+
+    idx = (idx + 1) % N_BW_EVENTS_TO_CACHE;
+  }
+
+  char *result = smartlist_join_strings(elements, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+
+  return result;
+}
+
+/** Called when we are sending a log message to the controllers: suspend
+ * sending further log messages to the controllers until we're done.  Used by
+ * CONN_LOG_PROTECT. */
+void
+disable_control_logging(void)
+{
+  ++disable_log_messages;
+}
+
+/** We're done sending a log message to the controllers: re-enable controller
+ * logging.  Used by CONN_LOG_PROTECT. */
+void
+enable_control_logging(void)
+{
+  if (--disable_log_messages < 0)
+    tor_assert(0);
+}
+
+/** We got a log message: tell any interested control connections. */
+void
+control_event_logmsg(int severity, uint32_t domain, const char *msg)
+{
+  int event;
+
+  /* Don't even think of trying to add stuff to a buffer from a cpuworker
+   * thread. (See #25987 for plan to fix.) */
+  if (! in_main_thread())
+    return;
+
+  if (disable_log_messages)
+    return;
+
+  if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) &&
+      severity <= LOG_NOTICE) {
+    char *esc = esc_for_log(msg);
+    ++disable_log_messages;
+    control_event_general_status(severity, "BUG REASON=%s", esc);
+    --disable_log_messages;
+    tor_free(esc);
+  }
+
+  event = log_severity_to_event(severity);
+  if (event >= 0 && EVENT_IS_INTERESTING(event)) {
+    char *b = NULL;
+    const char *s;
+    if (strchr(msg, '\n')) {
+      char *cp;
+      b = tor_strdup(msg);
+      for (cp = b; *cp; ++cp)
+        if (*cp == '\r' || *cp == '\n')
+          *cp = ' ';
+    }
+    switch (severity) {
+      case LOG_DEBUG: s = "DEBUG"; break;
+      case LOG_INFO: s = "INFO"; break;
+      case LOG_NOTICE: s = "NOTICE"; break;
+      case LOG_WARN: s = "WARN"; break;
+      case LOG_ERR: s = "ERR"; break;
+      default: s = "UnknownLogSeverity"; break;
+    }
+    ++disable_log_messages;
+    send_control_event(event,  "650 %s %s\r\n", s, b?b:msg);
+    if (severity == LOG_ERR) {
+      /* Force a flush, since we may be about to die horribly */
+      queued_events_flush_all(1);
+    }
+    --disable_log_messages;
+    tor_free(b);
+  }
+}
+
+/**
+ * Logging callback: called when there is a queued pending log callback.
+ */
+void
+control_event_logmsg_pending(void)
+{
+  if (! in_main_thread()) {
+    /* We can't handle this case yet, since we're using a
+     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
+     * use a different mechanism instead: see #25987.
+     **/
+    return;
+  }
+  tor_assert(flush_queued_events_event);
+  mainloop_event_activate(flush_queued_events_event);
+}
+
+/** Called whenever we receive new router descriptors: tell any
+ * interested control connections.  <b>routers</b> is a list of
+ * routerinfo_t's.
+ */
+int
+control_event_descriptors_changed(smartlist_t *routers)
+{
+  char *msg;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC))
+    return 0;
+
+  {
+    smartlist_t *names = smartlist_new();
+    char *ids;
+    SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
+        char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1);
+        router_get_verbose_nickname(b, ri);
+        smartlist_add(names, b);
+      });
+    ids = smartlist_join_strings(names, " ", 0, NULL);
+    tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids);
+    send_control_event_string(EVENT_NEW_DESC,  msg);
+    tor_free(ids);
+    tor_free(msg);
+    SMARTLIST_FOREACH(names, char *, cp, tor_free(cp));
+    smartlist_free(names);
+  }
+  return 0;
+}
+
+/** Called when an address mapping on <b>from</b> from changes to <b>to</b>.
+ * <b>expires</b> values less than 3 are special; see connection_edge.c.  If
+ * <b>error</b> is non-NULL, it is an error code describing the failure
+ * mode of the mapping.
+ */
+int
+control_event_address_mapped(const char *from, const char *to, time_t expires,
+                             const char *error, const int cached)
+{
+  if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP))
+    return 0;
+
+  if (expires < 3 || expires == TIME_MAX)
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s NEVER %s%s"
+                                "CACHED=\"%s\"\r\n",
+                                  from, to, error?error:"", error?" ":"",
+                                cached?"YES":"NO");
+  else {
+    char buf[ISO_TIME_LEN+1];
+    char buf2[ISO_TIME_LEN+1];
+    format_local_iso_time(buf,expires);
+    format_iso_time(buf2,expires);
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s \"%s\""
+                                " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n",
+                                from, to, buf,
+                                error?error:"", error?" ":"",
+                                buf2, cached?"YES":"NO");
+  }
+
+  return 0;
+}
+
+/** Cached liveness for network liveness events and GETINFO
+ */
+
+static int network_is_live = 0;
+
+static int
+get_cached_network_liveness(void)
+{
+  return network_is_live;
+}
+
+static void
+set_cached_network_liveness(int liveness)
+{
+  network_is_live = liveness;
+}
+
+/** The network liveness has changed; this is called from circuitstats.c
+ * whenever we receive a cell, or when timeout expires and we assume the
+ * network is down. */
+int
+control_event_network_liveness_update(int liveness)
+{
+  if (liveness > 0) {
+    if (get_cached_network_liveness() <= 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(1);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS UP\r\n");
+    }
+    /* else was already live, no-op */
+  } else {
+    if (get_cached_network_liveness() > 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(0);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS DOWN\r\n");
+    }
+    /* else was already dead, no-op */
+  }
+
+  return 0;
+}
+
+/** Helper function for NS-style events. Constructs and sends an event
+ * of type <b>event</b> with string <b>event_string</b> out of the set of
+ * networkstatuses <b>statuses</b>. Currently it is used for NS events
+ * and NEWCONSENSUS events. */
+static int
+control_event_networkstatus_changed_helper(smartlist_t *statuses,
+                                           uint16_t event,
+                                           const char *event_string)
+{
+  smartlist_t *strs;
+  char *s, *esc = NULL;
+  if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses))
+    return 0;
+
+  strs = smartlist_new();
+  smartlist_add_strdup(strs, "650+");
+  smartlist_add_strdup(strs, event_string);
+  smartlist_add_strdup(strs, "\r\n");
+  SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
+    {
+      s = networkstatus_getinfo_helper_single(rs);
+      if (!s) continue;
+      smartlist_add(strs, s);
+    });
+
+  s = smartlist_join_strings(strs, "", 0, NULL);
+  write_escaped_data(s, strlen(s), &esc);
+  SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp));
+  smartlist_free(strs);
+  tor_free(s);
+  send_control_event_string(event,  esc);
+  send_control_event_string(event,
+                            "650 OK\r\n");
+
+  tor_free(esc);
+  return 0;
+}
+
+/** Called when the routerstatus_ts <b>statuses</b> have changed: sends
+ * an NS event to any controller that cares. */
+int
+control_event_networkstatus_changed(smartlist_t *statuses)
+{
+  return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS");
+}
+
+/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS
+ * event consisting of an NS-style line for each relay in the consensus. */
+int
+control_event_newconsensus(const networkstatus_t *consensus)
+{
+  if (!control_event_is_interesting(EVENT_NEWCONSENSUS))
+    return 0;
+  return control_event_networkstatus_changed_helper(
+           consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS");
+}
+
+/** Called when we compute a new circuitbuildtimeout */
+int
+control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                               const char *args)
+{
+  const char *type_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET))
+    return 0;
+
+  switch (type) {
+    case BUILDTIMEOUT_SET_EVENT_COMPUTED:
+      type_string = "COMPUTED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESET:
+      type_string = "RESET";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_SUSPENDED:
+      type_string = "SUSPENDED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_DISCARD:
+      type_string = "DISCARD";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESUME:
+      type_string = "RESUME";
+      break;
+    default:
+      type_string = "UNKNOWN";
+      break;
+  }
+
+  send_control_event(EVENT_BUILDTIMEOUT_SET,
+                     "650 BUILDTIMEOUT_SET %s %s\r\n",
+                     type_string, args);
+
+  return 0;
+}
+
+/** Called when a signal has been processed from signal_callback */
+int
+control_event_signal(uintptr_t signal_num)
+{
+  const char *signal_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_GOT_SIGNAL))
+    return 0;
+
+  switch (signal_num) {
+    case SIGHUP:
+      signal_string = "RELOAD";
+      break;
+    case SIGUSR1:
+      signal_string = "DUMP";
+      break;
+    case SIGUSR2:
+      signal_string = "DEBUG";
+      break;
+    case SIGNEWNYM:
+      signal_string = "NEWNYM";
+      break;
+    case SIGCLEARDNSCACHE:
+      signal_string = "CLEARDNSCACHE";
+      break;
+    case SIGHEARTBEAT:
+      signal_string = "HEARTBEAT";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal",
+               (unsigned long)signal_num);
+      return -1;
+  }
+
+  send_control_event(EVENT_GOT_SIGNAL,  "650 SIGNAL %s\r\n",
+                     signal_string);
+  return 0;
+}
+
+/** Called when a single local_routerstatus_t has changed: Sends an NS event
+ * to any controller that cares. */
+int
+control_event_networkstatus_changed_single(const routerstatus_t *rs)
+{
+  smartlist_t *statuses;
+  int r;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NS))
+    return 0;
+
+  statuses = smartlist_new();
+  smartlist_add(statuses, (void*)rs);
+  r = control_event_networkstatus_changed(statuses);
+  smartlist_free(statuses);
+  return r;
+}
+
+/** Our own router descriptor has changed; tell any controllers that care.
+ */
+int
+control_event_my_descriptor_changed(void)
+{
+  send_control_event(EVENT_DESCCHANGED,  "650 DESCCHANGED\r\n");
+  return 0;
+}
+
+/** Helper: sends a status event where <b>type</b> is one of
+ * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where <b>severity</b> is one of
+ * LOG_{NOTICE,WARN,ERR}, and where <b>format</b> is a printf-style format
+ * string corresponding to <b>args</b>. */
+static int
+control_event_status(int type, int severity, const char *format, va_list args)
+{
+  char *user_buf = NULL;
+  char format_buf[160];
+  const char *status, *sev;
+
+  switch (type) {
+    case EVENT_STATUS_GENERAL:
+      status = "STATUS_GENERAL";
+      break;
+    case EVENT_STATUS_CLIENT:
+      status = "STATUS_CLIENT";
+      break;
+    case EVENT_STATUS_SERVER:
+      status = "STATUS_SERVER";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status type %d", type);
+      return -1;
+  }
+  switch (severity) {
+    case LOG_NOTICE:
+      sev = "NOTICE";
+      break;
+    case LOG_WARN:
+      sev = "WARN";
+      break;
+    case LOG_ERR:
+      sev = "ERR";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status severity %d", severity);
+      return -1;
+  }
+  if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s",
+                   status, sev)<0) {
+    log_warn(LD_BUG, "Format string too long.");
+    return -1;
+  }
+  tor_vasprintf(&user_buf, format, args);
+
+  send_control_event(type,  "%s %s\r\n", format_buf, user_buf);
+  tor_free(user_buf);
+  return 0;
+}
+
+#define CONTROL_EVENT_STATUS_BODY(event, sev)                   \
+  int r;                                                        \
+  do {                                                          \
+    va_list ap;                                                 \
+    if (!EVENT_IS_INTERESTING(event))                           \
+      return 0;                                                 \
+                                                                \
+    va_start(ap, format);                                       \
+    r = control_event_status((event), (sev), format, ap);       \
+    va_end(ap);                                                 \
+  } while (0)
+
+/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_general_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_general_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_client_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_client_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_server_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_server_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Called when the status of an entry guard with the given <b>nickname</b>
+ * and identity <b>digest</b> has changed to <b>status</b>: tells any
+ * controllers that care. */
+int
+control_event_guard(const char *nickname, const char *digest,
+                    const char *status)
+{
+  char hbuf[HEX_DIGEST_LEN+1];
+  base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN);
+  if (!EVENT_IS_INTERESTING(EVENT_GUARD))
+    return 0;
+
+  {
+    char buf[MAX_VERBOSE_NICKNAME_LEN+1];
+    const node_t *node = node_get_by_id(digest);
+    if (node) {
+      node_get_verbose_nickname(node, buf);
+    } else {
+      tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname);
+    }
+    send_control_event(EVENT_GUARD,
+                       "650 GUARD ENTRY %s %s\r\n", buf, status);
+  }
+  return 0;
+}
+
+/** Called when a configuration option changes. This is generally triggered
+ * by SETCONF requests and RELOAD/SIGHUP signals. The <b>elements</b> is
+ * a smartlist_t containing (key, value, ...) pairs in sequence.
+ * <b>value</b> can be NULL. */
+int
+control_event_conf_changed(const smartlist_t *elements)
+{
+  int i;
+  char *result;
+  smartlist_t *lines;
+  if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) ||
+      smartlist_len(elements) == 0) {
+    return 0;
+  }
+  lines = smartlist_new();
+  for (i = 0; i < smartlist_len(elements); i += 2) {
+    char *k = smartlist_get(elements, i);
+    char *v = smartlist_get(elements, i+1);
+    if (v == NULL) {
+      smartlist_add_asprintf(lines, "650-%s", k);
+    } else {
+      smartlist_add_asprintf(lines, "650-%s=%s", k, v);
+    }
+  }
+  result = smartlist_join_strings(lines, "\r\n", 0, NULL);
+  send_control_event(EVENT_CONF_CHANGED,
+    "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result);
+  tor_free(result);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+  return 0;
+}
+
+/** Helper: Return a newly allocated string containing a path to the
+ * file where we store our authentication cookie. */
+char *
+get_controller_cookie_file_name(void)
+{
+  const or_options_t *options = get_options();
+  if (options->CookieAuthFile && strlen(options->CookieAuthFile)) {
+    return tor_strdup(options->CookieAuthFile);
+  } else {
+    return get_datadir_fname("control_auth_cookie");
+  }
+}
+
+/* Initialize the cookie-based authentication system of the
+ * ControlPort. If <b>enabled</b> is 0, then disable the cookie
+ * authentication system.  */
+int
+init_control_cookie_authentication(int enabled)
+{
+  char *fname = NULL;
+  int retval;
+
+  if (!enabled) {
+    authentication_cookie_is_set = 0;
+    return 0;
+  }
+
+  fname = get_controller_cookie_file_name();
+  retval = init_cookie_authentication(fname, "", /* no header */
+                                      AUTHENTICATION_COOKIE_LEN,
+                                   get_options()->CookieAuthFileGroupReadable,
+                                      &authentication_cookie,
+                                      &authentication_cookie_is_set);
+  tor_free(fname);
+  return retval;
+}
+
+/** A copy of the process specifier of Tor's owning controller, or
+ * NULL if this Tor instance is not currently owned by a process. */
+static char *owning_controller_process_spec = NULL;
+
+/** A process-termination monitor for Tor's owning controller, or NULL
+ * if this Tor instance is not currently owned by a process. */
+static tor_process_monitor_t *owning_controller_process_monitor = NULL;
+
+/** Process-termination monitor callback for Tor's owning controller
+ * process. */
+static void
+owning_controller_procmon_cb(void *unused)
+{
+  (void)unused;
+
+  lost_owning_controller("process", "vanished");
+}
+
+/** Set <b>process_spec</b> as Tor's owning controller process.
+ * Exit on failure. */
+void
+monitor_owning_controller_process(const char *process_spec)
+{
+  const char *msg;
+
+  tor_assert((owning_controller_process_spec == NULL) ==
+             (owning_controller_process_monitor == NULL));
+
+  if (owning_controller_process_spec != NULL) {
+    if ((process_spec != NULL) && !strcmp(process_spec,
+                                          owning_controller_process_spec)) {
+      /* Same process -- return now, instead of disposing of and
+       * recreating the process-termination monitor. */
+      return;
+    }
+
+    /* We are currently owned by a process, and we should no longer be
+     * owned by it.  Free the process-termination monitor. */
+    tor_process_monitor_free(owning_controller_process_monitor);
+    owning_controller_process_monitor = NULL;
+
+    tor_free(owning_controller_process_spec);
+    owning_controller_process_spec = NULL;
+  }
+
+  tor_assert((owning_controller_process_spec == NULL) &&
+             (owning_controller_process_monitor == NULL));
+
+  if (process_spec == NULL)
+    return;
+
+  owning_controller_process_spec = tor_strdup(process_spec);
+  owning_controller_process_monitor =
+    tor_process_monitor_new(tor_libevent_get_base(),
+                            owning_controller_process_spec,
+                            LD_CONTROL,
+                            owning_controller_procmon_cb, NULL,
+                            &msg);
+
+  if (owning_controller_process_monitor == NULL) {
+    log_err(LD_BUG, "Couldn't create process-termination monitor for "
+            "owning controller: %s.  Exiting.",
+            msg);
+    owning_controller_process_spec = NULL;
+    tor_shutdown_event_loop_and_exit(1);
+  }
+}
+
+/** Convert the name of a bootstrapping phase <b>s</b> into strings
+ * <b>tag</b> and <b>summary</b> suitable for display by the controller. */
+static int
+bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
+                           const char **summary)
+{
+  switch (s) {
+    case BOOTSTRAP_STATUS_UNDEF:
+      *tag = "undef";
+      *summary = "Undefined";
+      break;
+    case BOOTSTRAP_STATUS_STARTING:
+      *tag = "starting";
+      *summary = "Starting";
+      break;
+    case BOOTSTRAP_STATUS_CONN_DIR:
+      *tag = "conn_dir";
+      *summary = "Connecting to directory server";
+      break;
+    case BOOTSTRAP_STATUS_HANDSHAKE:
+      *tag = "status_handshake";
+      *summary = "Finishing handshake";
+      break;
+    case BOOTSTRAP_STATUS_HANDSHAKE_DIR:
+      *tag = "handshake_dir";
+      *summary = "Finishing handshake with directory server";
+      break;
+    case BOOTSTRAP_STATUS_ONEHOP_CREATE:
+      *tag = "onehop_create";
+      *summary = "Establishing an encrypted directory connection";
+      break;
+    case BOOTSTRAP_STATUS_REQUESTING_STATUS:
+      *tag = "requesting_status";
+      *summary = "Asking for networkstatus consensus";
+      break;
+    case BOOTSTRAP_STATUS_LOADING_STATUS:
+      *tag = "loading_status";
+      *summary = "Loading networkstatus consensus";
+      break;
+    case BOOTSTRAP_STATUS_LOADING_KEYS:
+      *tag = "loading_keys";
+      *summary = "Loading authority key certs";
+      break;
+    case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS:
+      *tag = "requesting_descriptors";
+      /* XXXX this appears to incorrectly report internal on most loads */
+      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
+        "Asking for relay descriptors for internal paths" :
+        "Asking for relay descriptors";
+      break;
+    /* If we're sure there are no exits in the consensus,
+     * inform the controller by adding "internal"
+     * to the status summaries.
+     * (We only check this while loading descriptors,
+     * so we may not know in the earlier stages.)
+     * But if there are exits, we can't be sure whether
+     * we're creating internal or exit paths/circuits.
+     * XXXX Or should be use different tags or statuses
+     * for internal and exit/all? */
+    case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS:
+      *tag = "loading_descriptors";
+      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
+        "Loading relay descriptors for internal paths" :
+        "Loading relay descriptors";
+      break;
+    case BOOTSTRAP_STATUS_CONN_OR:
+      *tag = "conn_or";
+      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
+        "Connecting to the Tor network internally" :
+        "Connecting to the Tor network";
+      break;
+    case BOOTSTRAP_STATUS_HANDSHAKE_OR:
+      *tag = "handshake_or";
+      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
+        "Finishing handshake with first hop of internal circuit" :
+        "Finishing handshake with first hop";
+      break;
+    case BOOTSTRAP_STATUS_CIRCUIT_CREATE:
+      *tag = "circuit_create";
+      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
+        "Establishing an internal Tor circuit" :
+        "Establishing a Tor circuit";
+      break;
+    case BOOTSTRAP_STATUS_DONE:
+      *tag = "done";
+      *summary = "Done";
+      break;
+    default:
+//      log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s);
+      *tag = *summary = "unknown";
+      return -1;
+  }
+  return 0;
+}
+
+/** What percentage through the bootstrap process are we? We remember
+ * this so we can avoid sending redundant bootstrap status events, and
+ * so we can guess context for the bootstrap messages which are
+ * ambiguous. It starts at 'undef', but gets set to 'starting' while
+ * Tor initializes. */
+static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+
+/** As bootstrap_percent, but holds the bootstrapping level at which we last
+ * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
+ * to avoid flooding the log with a new message every time we get a few more
+ * microdescriptors */
+static int notice_bootstrap_percent = 0;
+
+/** How many problems have we had getting to the next bootstrapping phase?
+ * These include failure to establish a connection to a Tor relay,
+ * failures to finish the TLS handshake, failures to validate the
+ * consensus document, etc. */
+static int bootstrap_problems = 0;
+
+/** We only tell the controller once we've hit a threshold of problems
+ * for the current phase. */
+#define BOOTSTRAP_PROBLEM_THRESHOLD 10
+
+/** When our bootstrapping progress level changes, but our bootstrapping
+ * status has not advanced, we only log at NOTICE when we have made at least
+ * this much progress.
+ */
+#define BOOTSTRAP_PCT_INCREMENT 5
+
+/** Called when Tor has made progress at bootstrapping its directory
+ * information and initial circuits.
+ *
+ * <b>status</b> is the new status, that is, what task we will be doing
+ * next. <b>progress</b> is zero if we just started this task, else it
+ * represents progress on the task.
+ *
+ * Return true if we logged a message at level NOTICE, and false otherwise.
+ */
+int
+control_event_bootstrap(bootstrap_status_t status, int progress)
+{
+  const char *tag, *summary;
+  char buf[BOOTSTRAP_MSG_LEN];
+
+  if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
+    return 0; /* already bootstrapped; nothing to be done here. */
+
+  /* special case for handshaking status, since our TLS handshaking code
+   * can't distinguish what the connection is going to be for. */
+  if (status == BOOTSTRAP_STATUS_HANDSHAKE) {
+    if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) {
+      status = BOOTSTRAP_STATUS_HANDSHAKE_DIR;
+    } else {
+      status = BOOTSTRAP_STATUS_HANDSHAKE_OR;
+    }
+  }
+
+  if (status > bootstrap_percent ||
+      (progress && progress > bootstrap_percent)) {
+    int loglevel = LOG_NOTICE;
+    bootstrap_status_to_string(status, &tag, &summary);
+
+    if (status <= bootstrap_percent &&
+        (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)) {
+      /* We log the message at info if the status hasn't advanced, and if less
+       * than BOOTSTRAP_PCT_INCREMENT progress has been made.
+       */
+      loglevel = LOG_INFO;
+    }
+
+    tor_log(loglevel, LD_CONTROL,
+            "Bootstrapped %d%%: %s", progress ? progress : status, summary);
+    tor_snprintf(buf, sizeof(buf),
+        "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
+        progress ? progress : status, tag, summary);
+    tor_snprintf(last_sent_bootstrap_message,
+                 sizeof(last_sent_bootstrap_message),
+                 "NOTICE %s", buf);
+    control_event_client_status(LOG_NOTICE, "%s", buf);
+    if (status > bootstrap_percent) {
+      bootstrap_percent = status; /* new milestone reached */
+    }
+    if (progress > bootstrap_percent) {
+      /* incremental progress within a milestone */
+      bootstrap_percent = progress;
+      bootstrap_problems = 0; /* Progress! Reset our problem counter. */
+    }
+    if (loglevel == LOG_NOTICE &&
+        bootstrap_percent > notice_bootstrap_percent) {
+      /* Remember that we gave a notice at this level. */
+      notice_bootstrap_percent = bootstrap_percent;
+    }
+    return loglevel == LOG_NOTICE;
+  }
+
+  return 0;
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a human-readable hint
+ * as to why, and <b>reason</b> provides a controller-facing short
+ * tag.  <b>conn</b> is the connection that caused this problem and
+ * can be NULL if a connection cannot be easily identified.
+ */
+void
+control_event_bootstrap_problem(const char *warn, const char *reason,
+                                const connection_t *conn, int dowarn)
+{
+  int status = bootstrap_percent;
+  const char *tag = "", *summary = "";
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *recommendation = "ignore";
+  int severity;
+  char *or_id = NULL, *hostaddr = NULL;
+  or_connection_t *or_conn = NULL;
+
+  /* bootstrap_percent must not be in "undefined" state here. */
+  tor_assert(status >= 0);
+
+  if (bootstrap_percent == 100)
+    return; /* already bootstrapped; nothing to be done here. */
+
+  bootstrap_problems++;
+
+  if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
+    dowarn = 1;
+
+  /* Don't warn about our bootstrapping status if we are hibernating or
+   * shutting down. */
+  if (we_are_hibernating())
+    dowarn = 0;
+
+  while (status>=0 && bootstrap_status_to_string(status, &tag, &summary) < 0)
+    status--; /* find a recognized status string based on current progress */
+  status = bootstrap_percent; /* set status back to the actual number */
+
+  severity = dowarn ? LOG_WARN : LOG_INFO;
+
+  if (dowarn)
+    recommendation = "warn";
+
+  if (conn && conn->type == CONN_TYPE_OR) {
+    /* XXX TO_OR_CONN can't deal with const */
+    or_conn = TO_OR_CONN((connection_t *)conn);
+    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
+  } else {
+    or_id = tor_strdup("?");
+  }
+
+  if (conn)
+    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
+  else
+    hostaddr = tor_strdup("?");
+
+  log_fn(severity,
+         LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; "
+         "count %d; recommendation %s; host %s at %s)",
+         status, summary, warn, reason,
+         bootstrap_problems, recommendation,
+         or_id, hostaddr);
+
+  connection_or_report_broken_states(severity, LD_HANDSHAKE);
+
+  tor_snprintf(buf, sizeof(buf),
+      "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
+      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
+      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
+      recommendation,
+      or_id, hostaddr);
+
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "WARN %s", buf);
+  control_event_client_status(LOG_WARN, "%s", buf);
+
+  tor_free(hostaddr);
+  tor_free(or_id);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a hint as to why, and
+ * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
+ * is the connection that caused this problem.
+ */
+MOCK_IMPL(void,
+control_event_bootstrap_prob_or, (const char *warn, int reason,
+                                  or_connection_t *or_conn))
+{
+  int dowarn = 0;
+
+  if (or_conn->have_noted_bootstrap_problem)
+    return;
+
+  or_conn->have_noted_bootstrap_problem = 1;
+
+  if (reason == END_OR_CONN_REASON_NO_ROUTE)
+    dowarn = 1;
+
+  /* If we are using bridges and all our OR connections are now
+     closed, it means that we totally failed to connect to our
+     bridges. Throw a warning. */
+  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
+    dowarn = 1;
+
+  control_event_bootstrap_problem(warn,
+                                  orconn_end_reason_to_control_string(reason),
+                                  TO_CONN(or_conn), dowarn);
+}
+
+/** We just generated a new summary of which countries we've seen clients
+ * from recently. Send a copy to the controller in case it wants to
+ * display it for the user. */
+void
+control_event_clients_seen(const char *controller_str)
+{
+  send_control_event(EVENT_CLIENTS_SEEN,
+    "650 CLIENTS_SEEN %s\r\n", controller_str);
+}
+
+/** A new pluggable transport called <b>transport_name</b> was
+ *  launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either
+ *  "server" or "client" depending on the mode of the pluggable
+ *  transport.
+ *  "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port
+ */
+void
+control_event_transport_launched(const char *mode, const char *transport_name,
+                                 tor_addr_t *addr, uint16_t port)
+{
+  send_control_event(EVENT_TRANSPORT_LAUNCHED,
+                     "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n",
+                     mode, transport_name, fmt_addr(addr), port);
+}
+
+/** Convert rendezvous auth type to string for HS_DESC control events
+ */
+const char *
+rend_auth_type_to_string(rend_auth_type_t auth_type)
+{
+  const char *str;
+
+  switch (auth_type) {
+    case REND_NO_AUTH:
+      str = "NO_AUTH";
+      break;
+    case REND_BASIC_AUTH:
+      str = "BASIC_AUTH";
+      break;
+    case REND_STEALTH_AUTH:
+      str = "STEALTH_AUTH";
+      break;
+    default:
+      str = "UNKNOWN";
+  }
+
+  return str;
+}
+
+/** Return a longname the node whose identity is <b>id_digest</b>. If
+ * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is
+ * returned instead.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+MOCK_IMPL(const char *,
+node_describe_longname_by_id,(const char *id_digest))
+{
+  static char longname[MAX_VERBOSE_NICKNAME_LEN+1];
+  node_get_verbose_nickname_by_id(id_digest, longname);
+  return longname;
+}
+
+/** Return either the onion address if the given pointer is a non empty
+ * string else the unknown string. */
+static const char *
+rend_hsaddress_str_or_unknown(const char *onion_address)
+{
+  static const char *str_unknown = "UNKNOWN";
+  const char *str_ret = str_unknown;
+
+  /* No valid pointer, unknown it is. */
+  if (!onion_address) {
+    goto end;
+  }
+  /* Empty onion address thus we don't know, unknown it is. */
+  if (onion_address[0] == '\0') {
+    goto end;
+  }
+  /* All checks are good so return the given onion address. */
+  str_ret = onion_address;
+
+ end:
+  return str_ret;
+}
+
+/** send HS_DESC requested event.
+ *
+ * <b>rend_query</b> is used to fetch requested onion address and auth type.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id_base32</b> is the ID of requested hs descriptor.
+ * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
+ */
+void
+control_event_hs_descriptor_requested(const char *onion_address,
+                                      rend_auth_type_t auth_type,
+                                      const char *id_digest,
+                                      const char *desc_id,
+                                      const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** For an HS descriptor query <b>rend_data</b>, using the
+ * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out
+ * which descriptor ID in the query is the right one.
+ *
+ * Return a pointer of the binary descriptor ID found in the query's object
+ * or NULL if not found. */
+static const char *
+get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
+{
+  int replica;
+  const char *desc_id = NULL;
+  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
+
+  /* Possible if the fetch was done using a descriptor ID. This means that
+   * the HSFETCH command was used. */
+  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
+    desc_id = rend_data_v2->desc_id_fetch;
+    goto end;
+  }
+
+  /* Without a directory fingerprint at this stage, we can't do much. */
+  if (hsdir_fp == NULL) {
+     goto end;
+  }
+
+  /* OK, we have an onion address so now let's find which descriptor ID
+   * is the one associated with the HSDir fingerprint. */
+  for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
+       replica++) {
+    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
+
+    SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
+      if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
+        /* Found it! This descriptor ID is the right one. */
+        desc_id = digest;
+        goto end;
+      }
+    } SMARTLIST_FOREACH_END(fingerprint);
+  }
+
+ end:
+  return desc_id;
+}
+
+/** send HS_DESC CREATED event when a local service generates a descriptor.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>desc_id</b> is the descriptor ID.
+ * <b>replica</b> is the the descriptor replica number. If it is negative, it
+ * is ignored.
+ */
+void
+control_event_hs_descriptor_created(const char *onion_address,
+                                    const char *desc_id,
+                                    int replica)
+{
+  char *replica_field = NULL;
+
+  if (BUG(!onion_address || !desc_id)) {
+    return;
+  }
+
+  if (replica >= 0) {
+    tor_asprintf(&replica_field, " REPLICA=%d", replica);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
+                     onion_address, desc_id,
+                     replica_field ? replica_field : "");
+  tor_free(replica_field);
+}
+
+/** send HS_DESC upload event.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id</b> is the ID of requested hs descriptor.
+ */
+void
+control_event_hs_descriptor_upload(const char *onion_address,
+                                   const char *id_digest,
+                                   const char *desc_id,
+                                   const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!onion_address || !id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
+                     onion_address,
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hsv2_descriptor_received
+ * control_event_hsv2_descriptor_failed
+ * control_event_hsv3_descriptor_failed
+ *
+ * So do not call this function directly.
+ */
+static void
+event_hs_descriptor_receive_end(const char *action,
+                                const char *onion_address,
+                                const char *desc_id,
+                                rend_auth_type_t auth_type,
+                                const char *hsdir_id_digest,
+                                const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !onion_address)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s %s %s%s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     desc_id ? desc_id : "",
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hs_descriptor_uploaded
+ * control_event_hs_descriptor_upload_failed
+ *
+ * So do not call this function directly.
+ */
+void
+control_event_hs_descriptor_upload_end(const char *action,
+                                       const char *onion_address,
+                                       const char *id_digest,
+                                       const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !id_digest)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s UNKNOWN %s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     node_describe_longname_by_id(id_digest),
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** send HS_DESC RECEIVED event
+ *
+ * called when we successfully received a hidden service descriptor.
+ */
+void
+control_event_hsv2_descriptor_received(const char *onion_address,
+                                       const rend_data_t *rend_data,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/* Send HS_DESC RECEIVED event
+ *
+ * Called when we successfully received a hidden service descriptor. */
+void
+control_event_hsv3_descriptor_received(const char *onion_address,
+                                       const char *desc_id,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/** send HS_DESC UPLOADED event
+ *
+ * called when we successfully uploaded a hidden service descriptor.
+ */
+void
+control_event_hs_descriptor_uploaded(const char *id_digest,
+                                     const char *onion_address)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+
+  control_event_hs_descriptor_upload_end("UPLOADED", onion_address,
+                                         id_digest, NULL);
+}
+
+/** Send HS_DESC event to inform controller that query <b>rend_data</b>
+ * failed to retrieve hidden service descriptor from directory identified by
+ * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
+ * add it to REASON= field.
+ */
+void
+control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
+                                  desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC event to inform controller that the query to
+ * <b>onion_address</b> failed to retrieve hidden service descriptor
+ * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
+ * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
+ * field. */
+void
+control_event_hsv3_descriptor_failed(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !reason)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC_CONTENT event after completion of a successful fetch from hs
+ * directory. If <b>hsdir_id_digest</b> is NULL, it is replaced by "UNKNOWN".
+ * If <b>content</b> is NULL, it is replaced by an empty string. The
+ * <b>onion_address</b> or <b>desc_id</b> set to NULL will no trigger the
+ * control event. */
+void
+control_event_hs_descriptor_content(const char *onion_address,
+                                    const char *desc_id,
+                                    const char *hsdir_id_digest,
+                                    const char *content)
+{
+  static const char *event_name = "HS_DESC_CONTENT";
+  char *esc_content = NULL;
+
+  if (!onion_address || !desc_id) {
+    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
+             onion_address, desc_id);
+    return;
+  }
+
+  if (content == NULL) {
+    /* Point it to empty content so it can still be escaped. */
+    content = "";
+  }
+  write_escaped_data(content, strlen(content), &esc_content);
+
+  send_control_event(EVENT_HS_DESC_CONTENT,
+                     "650+%s %s %s %s\r\n%s650 OK\r\n",
+                     event_name,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     desc_id,
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     esc_content);
+  tor_free(esc_content);
+}
+
+/** Send HS_DESC event to inform controller upload of hidden service
+ * descriptor identified by <b>id_digest</b> failed. If <b>reason</b>
+ * is not NULL, add it to REASON= field.
+ */
+void
+control_event_hs_descriptor_upload_failed(const char *id_digest,
+                                          const char *onion_address,
+                                          const char *reason)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+  control_event_hs_descriptor_upload_end("FAILED", onion_address,
+                                         id_digest, reason);
+}
+
+/** Free any leftover allocated memory of the control.c subsystem. */
+void
+control_free_all(void)
+{
+  smartlist_t *queued_events = NULL;
+
+  stats_prev_n_read = stats_prev_n_written = 0;
+
+  if (authentication_cookie) /* Free the auth cookie */
+    tor_free(authentication_cookie);
+  if (detached_onion_services) { /* Free the detached onion services */
+    SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
+    smartlist_free(detached_onion_services);
+  }
+
+  if (queued_control_events_lock) {
+    tor_mutex_acquire(queued_control_events_lock);
+    flush_queued_event_pending = 0;
+    queued_events = queued_control_events;
+    queued_control_events = NULL;
+    tor_mutex_release(queued_control_events_lock);
+  }
+  if (queued_events) {
+    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
+                      queued_event_free(ev));
+    smartlist_free(queued_events);
+  }
+  if (flush_queued_events_event) {
+    mainloop_event_free(flush_queued_events_event);
+    flush_queued_events_event = NULL;
+  }
+  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+  notice_bootstrap_percent = 0;
+  bootstrap_problems = 0;
+  authentication_cookie_is_set = 0;
+  global_event_mask = 0;
+  disable_log_messages = 0;
+  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
+}
+
+#ifdef TOR_UNIT_TESTS
+/* For testing: change the value of global_event_mask */
+void
+control_testing_set_global_event_mask(uint64_t mask)
+{
+  global_event_mask = mask;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/control/control.h b/src/feature/control/control.h
new file mode 100644
index 0000000000..5c5fe8a917
--- /dev/null
+++ b/src/feature/control/control.h
@@ -0,0 +1,415 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control.h
+ * \brief Header file for control.c.
+ **/
+
+#ifndef TOR_CONTROL_H
+#define TOR_CONTROL_H
+
+/** Used to indicate the type of a circuit event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum circuit_status_event_t {
+  CIRC_EVENT_LAUNCHED = 0,
+  CIRC_EVENT_BUILT    = 1,
+  CIRC_EVENT_EXTENDED = 2,
+  CIRC_EVENT_FAILED   = 3,
+  CIRC_EVENT_CLOSED   = 4,
+} circuit_status_event_t;
+
+/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
+ * The various types are defined in control-spec.txt . */
+typedef enum circuit_status_minor_event_t {
+  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+  CIRC_MINOR_EVENT_CANNIBALIZED,
+} circuit_status_minor_event_t;
+
+/** Used to indicate the type of a stream event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum stream_status_event_t {
+  STREAM_EVENT_SENT_CONNECT = 0,
+  STREAM_EVENT_SENT_RESOLVE = 1,
+  STREAM_EVENT_SUCCEEDED    = 2,
+  STREAM_EVENT_FAILED       = 3,
+  STREAM_EVENT_CLOSED       = 4,
+  STREAM_EVENT_NEW          = 5,
+  STREAM_EVENT_NEW_RESOLVE  = 6,
+  STREAM_EVENT_FAILED_RETRIABLE = 7,
+  STREAM_EVENT_REMAP        = 8
+} stream_status_event_t;
+
+/** Used to indicate the type of an OR connection event passed to the
+ * controller.  The various types are defined in control-spec.txt */
+typedef enum or_conn_status_event_t {
+  OR_CONN_EVENT_LAUNCHED     = 0,
+  OR_CONN_EVENT_CONNECTED    = 1,
+  OR_CONN_EVENT_FAILED       = 2,
+  OR_CONN_EVENT_CLOSED       = 3,
+  OR_CONN_EVENT_NEW          = 4,
+} or_conn_status_event_t;
+
+/** Used to indicate the type of a buildtime event */
+typedef enum buildtimeout_set_event_t {
+  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
+  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
+  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
+  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
+  BUILDTIMEOUT_SET_EVENT_RESUME = 4
+} buildtimeout_set_event_t;
+
+/** Enum describing various stages of bootstrapping, for use with controller
+ * bootstrap status events. The values range from 0 to 100. */
+typedef enum {
+  BOOTSTRAP_STATUS_UNDEF=-1,
+  BOOTSTRAP_STATUS_STARTING=0,
+  BOOTSTRAP_STATUS_CONN_DIR=5,
+  BOOTSTRAP_STATUS_HANDSHAKE=-2,
+  BOOTSTRAP_STATUS_HANDSHAKE_DIR=10,
+  BOOTSTRAP_STATUS_ONEHOP_CREATE=15,
+  BOOTSTRAP_STATUS_REQUESTING_STATUS=20,
+  BOOTSTRAP_STATUS_LOADING_STATUS=25,
+  BOOTSTRAP_STATUS_LOADING_KEYS=40,
+  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
+  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
+  BOOTSTRAP_STATUS_CONN_OR=80,
+  BOOTSTRAP_STATUS_HANDSHAKE_OR=85,
+  BOOTSTRAP_STATUS_CIRCUIT_CREATE=90,
+  BOOTSTRAP_STATUS_DONE=100
+} bootstrap_status_t;
+
+control_connection_t *TO_CONTROL_CONN(connection_t *);
+
+#define CONTROL_CONN_STATE_MIN_ 1
+/** State for a control connection: Authenticated and accepting v1 commands. */
+#define CONTROL_CONN_STATE_OPEN 1
+/** State for a control connection: Waiting for authentication; speaking
+ * protocol v1. */
+#define CONTROL_CONN_STATE_NEEDAUTH 2
+#define CONTROL_CONN_STATE_MAX_ 2
+
+/** Reason for remapping an AP connection's address: we have a cached
+ * answer. */
+#define REMAP_STREAM_SOURCE_CACHE 1
+/** Reason for remapping an AP connection's address: the exit node told us an
+ * answer. */
+#define REMAP_STREAM_SOURCE_EXIT 2
+
+void control_initialize_event_queue(void);
+
+void control_update_global_event_mask(void);
+void control_adjust_event_log_severity(void);
+
+void control_ports_write_to_file(void);
+
+/** Log information about the connection <b>conn</b>, protecting it as with
+ * CONN_LOG_PROTECT. Example:
+ *
+ * LOG_FN_CONN(conn, (LOG_DEBUG, "Socket %d wants to write", conn->s));
+ **/
+#define LOG_FN_CONN(conn, args)                 \
+  CONN_LOG_PROTECT(conn, log_fn args)
+
+#define CC_LOCAL_FD_IS_OWNER (1u<<0)
+#define CC_LOCAL_FD_IS_AUTHENTICATED (1u<<1)
+int control_connection_add_local_fd(tor_socket_t sock, unsigned flags);
+
+int connection_control_finished_flushing(control_connection_t *conn);
+int connection_control_reached_eof(control_connection_t *conn);
+void connection_control_closed(control_connection_t *conn);
+
+int connection_control_process_inbuf(control_connection_t *conn);
+
+#define EVENT_NS 0x000F
+int control_event_is_interesting(int event);
+
+void control_per_second_events(void);
+int control_any_per_second_event_enabled(void);
+
+int control_event_circuit_status(origin_circuit_t *circ,
+                                 circuit_status_event_t e, int reason);
+int control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                          int old_purpose);
+int control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                       int old_purpose,
+                                       const struct timeval *old_tv_created);
+int control_event_stream_status(entry_connection_t *conn,
+                                stream_status_event_t e,
+                                int reason);
+int control_event_or_conn_status(or_connection_t *conn,
+                                 or_conn_status_event_t e, int reason);
+int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
+int control_event_stream_bandwidth(edge_connection_t *edge_conn);
+int control_event_stream_bandwidth_used(void);
+int control_event_circ_bandwidth_used(void);
+int control_event_conn_bandwidth(connection_t *conn);
+int control_event_conn_bandwidth_used(void);
+int control_event_circuit_cell_stats(void);
+void control_event_logmsg(int severity, uint32_t domain, const char *msg);
+void control_event_logmsg_pending(void);
+int control_event_descriptors_changed(smartlist_t *routers);
+int control_event_address_mapped(const char *from, const char *to,
+                                 time_t expires, const char *error,
+                                 const int cached);
+int control_event_my_descriptor_changed(void);
+int control_event_network_liveness_update(int liveness);
+int control_event_networkstatus_changed(smartlist_t *statuses);
+
+int control_event_newconsensus(const networkstatus_t *consensus);
+int control_event_networkstatus_changed_single(const routerstatus_t *rs);
+int control_event_general_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_client_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_server_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+int control_event_general_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_client_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_server_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+
+int control_event_guard(const char *nickname, const char *digest,
+                        const char *status);
+int control_event_conf_changed(const smartlist_t *elements);
+int control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                                   const char *args);
+int control_event_signal(uintptr_t signal);
+
+int init_control_cookie_authentication(int enabled);
+char *get_controller_cookie_file_name(void);
+struct config_line_t;
+smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
+void disable_control_logging(void);
+void enable_control_logging(void);
+
+void monitor_owning_controller_process(const char *process_spec);
+
+int control_event_bootstrap(bootstrap_status_t status, int progress);
+MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
+                                                 int reason,
+                                                 or_connection_t *or_conn));
+void control_event_bootstrap_problem(const char *warn, const char *reason,
+                                     const connection_t *conn, int dowarn);
+
+void control_event_clients_seen(const char *controller_str);
+void control_event_transport_launched(const char *mode,
+                                      const char *transport_name,
+                                      tor_addr_t *addr, uint16_t port);
+const char *rend_auth_type_to_string(rend_auth_type_t auth_type);
+MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
+void control_event_hs_descriptor_requested(const char *onion_address,
+                                           rend_auth_type_t auth_type,
+                                           const char *id_digest,
+                                           const char *desc_id,
+                                           const char *hsdir_index);
+void control_event_hs_descriptor_created(const char *onion_address,
+                                         const char *desc_id,
+                                         int replica);
+void control_event_hs_descriptor_upload(const char *onion_address,
+                                        const char *desc_id,
+                                        const char *hs_dir,
+                                        const char *hsdir_index);
+void control_event_hs_descriptor_upload_end(const char *action,
+                                            const char *onion_address,
+                                            const char *hs_dir,
+                                            const char *reason);
+void control_event_hs_descriptor_uploaded(const char *hs_dir,
+                                          const char *onion_address);
+/* Hidden service v2 HS_DESC specific. */
+void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                          const char *id_digest,
+                                          const char *reason);
+void control_event_hsv2_descriptor_received(const char *onion_address,
+                                            const rend_data_t *rend_data,
+                                            const char *id_digest);
+/* Hidden service v3 HS_DESC specific. */
+void control_event_hsv3_descriptor_failed(const char *onion_address,
+                                          const char *desc_id,
+                                          const char *hsdir_id_digest,
+                                          const char *reason);
+void control_event_hsv3_descriptor_received(const char *onion_address,
+                                            const char *desc_id,
+                                            const char *hsdir_id_digest);
+void control_event_hs_descriptor_upload_failed(const char *hs_dir,
+                                               const char *onion_address,
+                                               const char *reason);
+void control_event_hs_descriptor_content(const char *onion_address,
+                                         const char *desc_id,
+                                         const char *hsdir_fp,
+                                         const char *content);
+
+void control_free_all(void);
+
+#ifdef CONTROL_PRIVATE
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/* Recognized asynchronous event types.  It's okay to expand this list
+ * because it is used both as a list of v0 event types, and as indices
+ * into the bitfield to determine which controllers want which events.
+ */
+/* This bitfield has no event zero    0x0000 */
+#define EVENT_MIN_                    0x0001
+#define EVENT_CIRCUIT_STATUS          0x0001
+#define EVENT_STREAM_STATUS           0x0002
+#define EVENT_OR_CONN_STATUS          0x0003
+#define EVENT_BANDWIDTH_USED          0x0004
+#define EVENT_CIRCUIT_STATUS_MINOR    0x0005
+#define EVENT_NEW_DESC                0x0006
+#define EVENT_DEBUG_MSG               0x0007
+#define EVENT_INFO_MSG                0x0008
+#define EVENT_NOTICE_MSG              0x0009
+#define EVENT_WARN_MSG                0x000A
+#define EVENT_ERR_MSG                 0x000B
+#define EVENT_ADDRMAP                 0x000C
+/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
+   can reclaim 0x000D. */
+#define EVENT_DESCCHANGED             0x000E
+/* Exposed above */
+// #define EVENT_NS                   0x000F
+#define EVENT_STATUS_CLIENT           0x0010
+#define EVENT_STATUS_SERVER           0x0011
+#define EVENT_STATUS_GENERAL          0x0012
+#define EVENT_GUARD                   0x0013
+#define EVENT_STREAM_BANDWIDTH_USED   0x0014
+#define EVENT_CLIENTS_SEEN            0x0015
+#define EVENT_NEWCONSENSUS            0x0016
+#define EVENT_BUILDTIMEOUT_SET        0x0017
+#define EVENT_GOT_SIGNAL              0x0018
+#define EVENT_CONF_CHANGED            0x0019
+#define EVENT_CONN_BW                 0x001A
+#define EVENT_CELL_STATS              0x001B
+/* UNUSED :                           0x001C */
+#define EVENT_CIRC_BANDWIDTH_USED     0x001D
+#define EVENT_TRANSPORT_LAUNCHED      0x0020
+#define EVENT_HS_DESC                 0x0021
+#define EVENT_HS_DESC_CONTENT         0x0022
+#define EVENT_NETWORK_LIVENESS        0x0023
+#define EVENT_MAX_                    0x0023
+
+/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */
+#define EVENT_CAPACITY_               0x0040
+
+/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a
+ * different structure, as it can only handle a maximum left shift of 1<<63. */
+
+#if EVENT_MAX_ >= EVENT_CAPACITY_
+#error control_connection_t.event_mask has an event greater than its capacity
+#endif
+
+#define EVENT_MASK_(e)               (((uint64_t)1)<<(e))
+
+#define EVENT_MASK_NONE_             ((uint64_t)0x0)
+
+#define EVENT_MASK_ABOVE_MIN_        ((~((uint64_t)0x0)) << EVENT_MIN_)
+#define EVENT_MASK_BELOW_MAX_        ((~((uint64_t)0x0)) \
+                                      >> (EVENT_CAPACITY_ - EVENT_MAX_ \
+                                          - EVENT_MIN_))
+
+#define EVENT_MASK_ALL_              (EVENT_MASK_ABOVE_MIN_ \
+                                      & EVENT_MASK_BELOW_MAX_)
+
+/* Used only by control.c and test.c */
+STATIC size_t write_escaped_data(const char *data, size_t len, char **out);
+STATIC size_t read_escaped_data(const char *data, size_t len, char **out);
+
+#ifdef TOR_UNIT_TESTS
+MOCK_DECL(STATIC void,
+          send_control_event_string,(uint16_t event, const char *msg));
+
+MOCK_DECL(STATIC void,
+          queue_control_event_string,(uint16_t event, char *msg));
+
+void control_testing_set_global_event_mask(uint64_t mask);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/** Helper structure: temporarily stores cell statistics for a circuit. */
+typedef struct cell_stats_t {
+  /** Number of cells added in app-ward direction by command. */
+  uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells added in exit-ward direction by command. */
+  uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in app-ward direction by command. */
+  uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in exit-ward direction by command. */
+  uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in app-ward direction by command. */
+  uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in exit-ward direction by command. */
+  uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1];
+} cell_stats_t;
+void sum_up_cell_stats_by_command(circuit_t *circ,
+                                  cell_stats_t *cell_stats);
+void append_cell_stats_by_command(smartlist_t *event_parts,
+                                  const char *key,
+                                  const uint64_t *include_if_non_zero,
+                                  const uint64_t *number_to_include);
+void format_cell_stats(char **event_string, circuit_t *circ,
+                       cell_stats_t *cell_stats);
+STATIC char *get_bw_samples(void);
+
+/* ADD_ONION secret key to create an ephemeral service. The command supports
+ * multiple versions so this union stores the key and passes it to the HS
+ * subsystem depending on the requested version. */
+typedef union add_onion_secret_key_t {
+  /* Hidden service v2 secret key. */
+  crypto_pk_t *v2;
+  /* Hidden service v3 secret key. */
+  ed25519_secret_key_t *v3;
+} add_onion_secret_key_t;
+
+STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
+                                   const char **key_new_alg_out,
+                                   char **key_new_blob_out,
+                                   add_onion_secret_key_t *decoded_key,
+                                   int *hs_version, char **err_msg_out);
+
+STATIC rend_authorized_client_t *
+add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out);
+
+STATIC int getinfo_helper_onions(
+    control_connection_t *control_conn,
+    const char *question,
+    char **answer,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_networkstatus(
+    const char *flavor,
+    download_status_t **dl_to_emit,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_cert(
+    const char *fp_sk_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_desc(
+    const char *desc_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_bridge(
+    const char *bridge_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC int getinfo_helper_downloads(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_dir(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_current_time(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+
+#endif /* defined(CONTROL_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_H) */
diff --git a/src/feature/control/control_connection_st.h b/src/feature/control/control_connection_st.h
new file mode 100644
index 0000000000..4f8ab25d99
--- /dev/null
+++ b/src/feature/control/control_connection_st.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CONTROL_CONNECTION_ST_H
+#define CONTROL_CONNECTION_ST_H
+
+#include "or/or.h"
+#include "or/connection_st.h"
+
+/** Subtype of connection_t for an connection to a controller. */
+struct control_connection_t {
+  connection_t base_;
+
+  uint64_t event_mask; /**< Bitfield: which events does this controller
+                        * care about?
+                        * EVENT_MAX_ is >31, so we need a 64 bit mask */
+
+  /** True if we have sent a protocolinfo reply on this connection. */
+  unsigned int have_sent_protocolinfo:1;
+  /** True if we have received a takeownership command on this
+   * connection. */
+  unsigned int is_owning_control_connection:1;
+
+  /** List of ephemeral onion services belonging to this connection. */
+  smartlist_t *ephemeral_onion_services;
+
+  /** If we have sent an AUTHCHALLENGE reply on this connection and
+   * have not received a successful AUTHENTICATE command, points to
+   * the value which the client must send to authenticate itself;
+   * otherwise, NULL. */
+  char *safecookie_client_hash;
+
+  /** Amount of space allocated in incoming_cmd. */
+  uint32_t incoming_cmd_len;
+  /** Number of bytes currently stored in incoming_cmd. */
+  uint32_t incoming_cmd_cur_len;
+  /** A control command that we're reading from the inbuf, but which has not
+   * yet arrived completely. */
+  char *incoming_cmd;
+};
+
+#endif
+
diff --git a/src/feature/dirauth/dircollate.c b/src/feature/dirauth/dircollate.c
new file mode 100644
index 0000000000..246977dcc8
--- /dev/null
+++ b/src/feature/dirauth/dircollate.c
@@ -0,0 +1,327 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dircollate.c
+ *
+ * \brief Collation code for figuring out which identities to vote for in
+ *   the directory voting process.
+ *
+ * During the consensus calculation, when an authority is looking at the vote
+ * documents from all the authorities, it needs to compute the consensus for
+ * each relay listed by at least one authority.  But the notion of "each
+ * relay" can be tricky: some relays have Ed25519 keys, and others don't.
+ *
+ * Moreover, older consensus methods did RSA-based ID collation alone, and
+ * ignored Ed25519 keys.  We need to support those too until we're completely
+ * sure that authorities will never downgrade.
+ *
+ * This module is invoked exclusively from dirvote.c.
+ */
+
+#define DIRCOLLATE_PRIVATE
+#include "or/dirauth/dircollate.h"
+#include "or/dirauth/dirvote.h"
+
+#include "or/networkstatus_st.h"
+#include "or/vote_routerstatus_st.h"
+
+static void dircollator_collate_by_ed25519(dircollator_t *dc);
+
+/** Hashtable entry mapping a pair of digests (actually an ed25519 key and an
+ * RSA SHA1 digest) to an array of vote_routerstatus_t. */
+typedef struct ddmap_entry_s {
+  HT_ENTRY(ddmap_entry_s) node;
+  /** A SHA1-RSA1024 identity digest and Ed25519 identity key,
+   * concatenated.  (If there is no ed25519 identity key, there is no
+   * entry in this table.) */
+  uint8_t d[DIGEST_LEN + DIGEST256_LEN];
+  /* The nth member of this array corresponds to the vote_routerstatus_t (if
+   * any) received for this digest pair from the nth voter. */
+  vote_routerstatus_t *vrs_lst[FLEXIBLE_ARRAY_MEMBER];
+} ddmap_entry_t;
+
+#define ddmap_entry_free(e) \
+  FREE_AND_NULL(ddmap_entry_t, ddmap_entry_free_, (e))
+
+/** Release all storage held by e. */
+static void
+ddmap_entry_free_(ddmap_entry_t *e)
+{
+  tor_free(e);
+}
+
+/** Return a new empty ddmap_entry, with <b>n_votes</b> elements in
+ * vrs_list. */
+static ddmap_entry_t *
+ddmap_entry_new(int n_votes)
+{
+  return tor_malloc_zero(offsetof(ddmap_entry_t, vrs_lst) +
+                         sizeof(vote_routerstatus_t *) * n_votes);
+}
+
+/** Helper: compute a hash of a single ddmap_entry_t's identity (or
+ * identities) */
+static unsigned
+ddmap_entry_hash(const ddmap_entry_t *ent)
+{
+  return (unsigned) siphash24g(ent->d, sizeof(ent->d));
+}
+
+/** Helper: return true if <b>a</b> and <b>b</b> have the same
+ * identity/identities. */
+static unsigned
+ddmap_entry_eq(const ddmap_entry_t *a, const ddmap_entry_t *b)
+{
+  return fast_memeq(a->d, b->d, sizeof(a->d));
+}
+
+/** Record the RSA identity of <b>ent</b> as <b>rsa_sha1</b>, and the
+ * ed25519 identity as <b>ed25519</b>.  Both must be provided. */
+static void
+ddmap_entry_set_digests(ddmap_entry_t *ent,
+                        const uint8_t *rsa_sha1,
+                        const uint8_t *ed25519)
+{
+  memcpy(ent->d, rsa_sha1, DIGEST_LEN);
+  memcpy(ent->d + DIGEST_LEN, ed25519, DIGEST256_LEN);
+}
+
+HT_PROTOTYPE(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash,
+             ddmap_entry_eq)
+HT_GENERATE2(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash,
+             ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_)
+
+/** Helper: add a single vote_routerstatus_t <b>vrs</b> to the collator
+ * <b>dc</b>, indexing it by its RSA key digest, and by the 2-tuple of its RSA
+ * key digest and Ed25519 key.   It must come from the <b>vote_num</b>th
+ * vote.
+ *
+ * Requires that the vote is well-formed -- that is, that it has no duplicate
+ * routerstatus entries.  We already checked for that when parsing the vote. */
+static void
+dircollator_add_routerstatus(dircollator_t *dc,
+                             int vote_num,
+                             networkstatus_t *vote,
+                             vote_routerstatus_t *vrs)
+{
+  const char *id = vrs->status.identity_digest;
+
+  /* Clear this flag; we might set it later during the voting process */
+  vrs->ed25519_reflects_consensus = 0;
+
+  (void) vote; // We don't currently need this.
+
+  /* First, add this item to the appropriate RSA-SHA-Id array. */
+  vote_routerstatus_t **vrs_lst = digestmap_get(dc->by_rsa_sha1, id);
+  if (NULL == vrs_lst) {
+    vrs_lst = tor_calloc(dc->n_votes, sizeof(vote_routerstatus_t *));
+    digestmap_set(dc->by_rsa_sha1, id, vrs_lst);
+  }
+  tor_assert(vrs_lst[vote_num] == NULL);
+  vrs_lst[vote_num] = vrs;
+
+  const uint8_t *ed = vrs->ed25519_id;
+
+  if (! vrs->has_ed25519_listing)
+    return;
+
+  /* Now add it to the appropriate <Ed,RSA-SHA-Id> array. */
+  ddmap_entry_t search, *found;
+  memset(&search, 0, sizeof(search));
+  ddmap_entry_set_digests(&search, (const uint8_t *)id, ed);
+  found = HT_FIND(double_digest_map, &dc->by_both_ids, &search);
+  if (NULL == found) {
+    found = ddmap_entry_new(dc->n_votes);
+    ddmap_entry_set_digests(found, (const uint8_t *)id, ed);
+    HT_INSERT(double_digest_map, &dc->by_both_ids, found);
+  }
+  vrs_lst = found->vrs_lst;
+  tor_assert(vrs_lst[vote_num] == NULL);
+  vrs_lst[vote_num] = vrs;
+}
+
+/** Create and return a new dircollator object to use when collating
+ * <b>n_votes</b> out of a total of <b>n_authorities</b>. */
+dircollator_t *
+dircollator_new(int n_votes, int n_authorities)
+{
+  dircollator_t *dc = tor_malloc_zero(sizeof(dircollator_t));
+
+  tor_assert(n_votes <= n_authorities);
+
+  dc->n_votes = n_votes;
+  dc->n_authorities = n_authorities;
+
+  dc->by_rsa_sha1 = digestmap_new();
+  HT_INIT(double_digest_map, &dc->by_both_ids);
+
+  return dc;
+}
+
+/** Release all storage held by <b>dc</b>. */
+void
+dircollator_free_(dircollator_t *dc)
+{
+  if (!dc)
+    return;
+
+  if (dc->by_collated_rsa_sha1 != dc->by_rsa_sha1)
+    digestmap_free(dc->by_collated_rsa_sha1, NULL);
+
+  digestmap_free(dc->by_rsa_sha1, tor_free_);
+  smartlist_free(dc->all_rsa_sha1_lst);
+
+  ddmap_entry_t **e, **next, *this;
+  for (e = HT_START(double_digest_map, &dc->by_both_ids);
+       e != NULL; e = next) {
+    this = *e;
+    next = HT_NEXT_RMV(double_digest_map, &dc->by_both_ids, e);
+    ddmap_entry_free(this);
+  }
+  HT_CLEAR(double_digest_map, &dc->by_both_ids);
+
+  tor_free(dc);
+}
+
+/** Add a single vote <b>v</b> to a dircollator <b>dc</b>.  This function must
+ * be called exactly once for each vote to be used in the consensus. It may
+ * only be called before dircollator_collate().
+ */
+void
+dircollator_add_vote(dircollator_t *dc, networkstatus_t *v)
+{
+  tor_assert(v->type == NS_TYPE_VOTE);
+  tor_assert(dc->next_vote_num < dc->n_votes);
+  tor_assert(!dc->is_collated);
+
+  const int votenum = dc->next_vote_num++;
+
+  SMARTLIST_FOREACH_BEGIN(v->routerstatus_list, vote_routerstatus_t *, vrs) {
+    dircollator_add_routerstatus(dc, votenum, v, vrs);
+  } SMARTLIST_FOREACH_END(vrs);
+}
+
+/** Sort the entries in <b>dc</b> according to <b>consensus_method</b>, so
+ * that the consensus process can iterate over them with
+ * dircollator_n_routers() and dircollator_get_votes_for_router(). */
+void
+dircollator_collate(dircollator_t *dc, int consensus_method)
+{
+  (void) consensus_method;
+
+  tor_assert(!dc->is_collated);
+  dc->all_rsa_sha1_lst = smartlist_new();
+
+  dircollator_collate_by_ed25519(dc);
+
+  smartlist_sort_digests(dc->all_rsa_sha1_lst);
+  dc->is_collated = 1;
+}
+
+/**
+ * Collation function for ed25519 consensuses: collate the votes for each
+ * entry in <b>dc</b> by ed25519 key and by RSA key.
+ *
+ * The rule is, approximately:
+ *    If a (ed,rsa) identity is listed by more than half of authorities,
+ *    include it.  And include all (rsa)-only votes about that node as
+ *    matching.
+ *
+ *    Otherwise, if an (*,rsa) or (rsa) identity is listed by more than
+ *    half of the authorities, and no (ed,rsa) pair for the same RSA key
+ *    has been already been included based on the rule above, include
+ *    that RSA identity.
+ */
+static void
+dircollator_collate_by_ed25519(dircollator_t *dc)
+{
+  const int total_authorities = dc->n_authorities;
+  digestmap_t *rsa_digests = digestmap_new();
+
+  ddmap_entry_t **iter;
+
+  /* Go over all <ed,rsa> pairs */
+  HT_FOREACH(iter, double_digest_map, &dc->by_both_ids) {
+    ddmap_entry_t *ent = *iter;
+    int n = 0, i;
+    for (i = 0; i < dc->n_votes; ++i) {
+      if (ent->vrs_lst[i] != NULL)
+        ++n;
+    }
+
+    /* If not enough authorties listed this exact <ed,rsa> pair,
+     * don't include it. */
+    if (n <= total_authorities / 2)
+      continue;
+
+    /* Now consider whether there are any other entries with the same
+     * RSA key (but with possibly different or missing ed value). */
+    vote_routerstatus_t **vrs_lst2 = digestmap_get(dc->by_rsa_sha1,
+                                                   (char*)ent->d);
+    tor_assert(vrs_lst2);
+
+    for (i = 0; i < dc->n_votes; ++i) {
+      if (ent->vrs_lst[i] != NULL) {
+        ent->vrs_lst[i]->ed25519_reflects_consensus = 1;
+      } else if (vrs_lst2[i] && ! vrs_lst2[i]->has_ed25519_listing) {
+        ent->vrs_lst[i] = vrs_lst2[i];
+      }
+    }
+
+    /* Record that we have seen this RSA digest. */
+    digestmap_set(rsa_digests, (char*)ent->d, ent->vrs_lst);
+    smartlist_add(dc->all_rsa_sha1_lst, ent->d);
+  }
+
+  /* Now look over all entries with an RSA digest, looking for RSA digests
+   * we didn't put in yet.
+   */
+  DIGESTMAP_FOREACH(dc->by_rsa_sha1, k, vote_routerstatus_t **, vrs_lst) {
+    if (digestmap_get(rsa_digests, k) != NULL)
+      continue; /* We already included this RSA digest */
+
+    int n = 0, i;
+    for (i = 0; i < dc->n_votes; ++i) {
+      if (vrs_lst[i] != NULL)
+        ++n;
+    }
+
+    if (n <= total_authorities / 2)
+      continue; /* Not enough votes */
+
+    digestmap_set(rsa_digests, k, vrs_lst);
+    smartlist_add(dc->all_rsa_sha1_lst, (char *)k);
+  } DIGESTMAP_FOREACH_END;
+
+  dc->by_collated_rsa_sha1 = rsa_digests;
+}
+
+/** Return the total number of collated router entries.  This function may
+ * only be called after dircollator_collate. */
+int
+dircollator_n_routers(dircollator_t *dc)
+{
+  tor_assert(dc->is_collated);
+  return smartlist_len(dc->all_rsa_sha1_lst);
+}
+
+/** Return an array of vote_routerstatus_t entries for the <b>idx</b>th router
+ * in the collation order.  Each array contains n_votes elements, where the
+ * nth element of the array is the vote_routerstatus_t from the nth voter for
+ * this identity (or NULL if there is no such entry).
+ *
+ * The maximum value for <b>idx</b> is dircollator_n_routers().
+ *
+ * This function may only be called after dircollator_collate. */
+vote_routerstatus_t **
+dircollator_get_votes_for_router(dircollator_t *dc, int idx)
+{
+  tor_assert(dc->is_collated);
+  tor_assert(idx < smartlist_len(dc->all_rsa_sha1_lst));
+  return digestmap_get(dc->by_collated_rsa_sha1,
+                       smartlist_get(dc->all_rsa_sha1_lst, idx));
+}
+
diff --git a/src/feature/dirauth/dircollate.h b/src/feature/dirauth/dircollate.h
new file mode 100644
index 0000000000..aae7829786
--- /dev/null
+++ b/src/feature/dirauth/dircollate.h
@@ -0,0 +1,70 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dircollate.h
+ * \brief Header file for dircollate.c.
+ **/
+
+#ifndef TOR_DIRCOLLATE_H
+#define TOR_DIRCOLLATE_H
+
+#include "lib/testsupport/testsupport.h"
+#include "or/or.h"
+
+typedef struct dircollator_s dircollator_t;
+
+dircollator_t *dircollator_new(int n_votes, int n_authorities);
+void dircollator_free_(dircollator_t *obj);
+#define dircollator_free(c) \
+  FREE_AND_NULL(dircollator_t, dircollator_free_, (c))
+void dircollator_add_vote(dircollator_t *dc, networkstatus_t *v);
+
+void dircollator_collate(dircollator_t *dc, int consensus_method);
+
+int dircollator_n_routers(dircollator_t *dc);
+vote_routerstatus_t **dircollator_get_votes_for_router(dircollator_t *dc,
+                                                       int idx);
+
+#ifdef DIRCOLLATE_PRIVATE
+struct ddmap_entry_s;
+typedef HT_HEAD(double_digest_map, ddmap_entry_s) double_digest_map_t;
+/** A dircollator keeps track of all the routerstatus entries in a
+ * set of networkstatus votes, and matches them by an appropriate rule. */
+struct dircollator_s {
+  /** True iff we have run the collation algorithm. */
+  int is_collated;
+  /** The total number of votes that we received. */
+  int n_votes;
+  /** The total number of authorities we acknowledge. */
+  int n_authorities;
+
+  /** The index which the next vote to be added to this collator should
+   * receive. */
+  int next_vote_num;
+  /** Map from RSA-SHA1 identity digest to an array of <b>n_votes</b>
+   * vote_routerstatus_t* pointers, such that the i'th member of the
+   * array is the i'th vote's entry for that RSA-SHA1 ID.*/
+  digestmap_t *by_rsa_sha1;
+  /** Map from <ed, RSA-SHA1> pair to an array similar to that used in
+   * by_rsa_sha1 above. We include <NULL,RSA-SHA1> entries for votes that
+   * say that there is no Ed key. */
+  struct double_digest_map by_both_ids;
+
+  /** One of two outputs created by collation: a map from RSA-SHA1
+   * identity digest to an array of the vote_routerstatus_t objects.  Entries
+   * only exist in this map for identities that we should include in the
+   * consensus. */
+  digestmap_t *by_collated_rsa_sha1;
+
+  /** One of two outputs created by collation: a sorted array of RSA-SHA1
+   * identity digests .*/
+  smartlist_t *all_rsa_sha1_lst;
+};
+#endif /* defined(DIRCOLLATE_PRIVATE) */
+
+#endif /* !defined(TOR_DIRCOLLATE_H) */
+
diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c
new file mode 100644
index 0000000000..15f8378744
--- /dev/null
+++ b/src/feature/dirauth/dirvote.c
@@ -0,0 +1,4568 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DIRVOTE_PRIVATE
+#include "or/or.h"
+#include "or/config.h"
+#include "or/dirauth/dircollate.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/parsecommon.h"
+#include "or/policies.h"
+#include "or/protover.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerkeys.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/entrynodes.h" /* needed for guardfraction methods */
+#include "or/torcert.h"
+#include "or/voting_schedule.h"
+
+#include "or/dirauth/dirvote.h"
+#include "or/dirauth/mode.h"
+#include "or/dirauth/shared_random_state.h"
+
+#include "or/authority_cert_st.h"
+#include "or/cached_dir_st.h"
+#include "or/dir_server_st.h"
+#include "or/document_signature_st.h"
+#include "or/microdesc_st.h"
+#include "or/networkstatus_st.h"
+#include "or/networkstatus_voter_info_st.h"
+#include "or/node_st.h"
+#include "or/ns_detached_signatures_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/vote_microdesc_hash_st.h"
+#include "or/vote_routerstatus_st.h"
+#include "or/vote_timing_st.h"
+
+#include "lib/container/order.h"
+#include "lib/encoding/confline.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+/**
+ * \file dirvote.c
+ * \brief Functions to compute directory consensus, and schedule voting.
+ *
+ * This module is the center of the consensus-voting based directory
+ * authority system.  With this system, a set of authorities first
+ * publish vote based on their opinions of the network, and then compute
+ * a consensus from those votes.  Each authority signs the consensus,
+ * and clients trust the consensus if enough known authorities have
+ * signed it.
+ *
+ * The code in this module is only invoked on directory authorities.  It's
+ * responsible for:
+ *
+ * <ul>
+ *   <li>Generating this authority's vote networkstatus, based on the
+ *       authority's view of the network as represented in dirserv.c
+ *   <li>Formatting the vote networkstatus objects.
+ *   <li>Generating the microdescriptors that correspond to our own
+ *       vote.
+ *   <li>Sending votes to all the other authorities.
+ *   <li>Trying to fetch missing votes from other authorities.
+ *   <li>Computing the consensus from a set of votes, as well as
+ *       a "detached signature" object for other authorities to fetch.
+ *   <li>Collecting other authorities' signatures on the same consensus,
+ *       until there are enough.
+ *   <li>Publishing the consensus to the reset of the directory system.
+ *   <li>Scheduling all of the above operations.
+ * </ul>
+ *
+ * The main entry points are in dirvote_act(), which handles scheduled
+ * actions; and dirvote_add_vote() and dirvote_add_signatures(), which
+ * handle uploaded and downloaded votes and signatures.
+ *
+ * (See dir-spec.txt from torspec.git for a complete specification of
+ * the directory protocol and voting algorithms.)
+ **/
+
+/** A consensus that we have built and are appending signatures to.  Once it's
+ * time to publish it, it will become an active consensus if it accumulates
+ * enough signatures. */
+typedef struct pending_consensus_t {
+  /** The body of the consensus that we're currently building.  Once we
+   * have it built, it goes into dirserv.c */
+  char *body;
+  /** The parsed in-progress consensus document. */
+  networkstatus_t *consensus;
+} pending_consensus_t;
+
+/* DOCDOC dirvote_add_signatures_to_all_pending_consensuses */
+static int dirvote_add_signatures_to_all_pending_consensuses(
+                       const char *detached_signatures_body,
+                       const char *source,
+                       const char **msg_out);
+static int dirvote_add_signatures_to_pending_consensus(
+                       pending_consensus_t *pc,
+                       ns_detached_signatures_t *sigs,
+                       const char *source,
+                       int severity,
+                       const char **msg_out);
+static char *list_v3_auth_ids(void);
+static void dirvote_fetch_missing_votes(void);
+static void dirvote_fetch_missing_signatures(void);
+static int dirvote_perform_vote(void);
+static void dirvote_clear_votes(int all_votes);
+static int dirvote_compute_consensuses(void);
+static int dirvote_publish_consensus(void);
+
+/* =====
+ * Certificate functions
+ * ===== */
+
+/** Allocate and return a new authority_cert_t with the same contents as
+ * <b>cert</b>. */
+STATIC authority_cert_t *
+authority_cert_dup(authority_cert_t *cert)
+{
+  authority_cert_t *out = tor_malloc(sizeof(authority_cert_t));
+  tor_assert(cert);
+
+  memcpy(out, cert, sizeof(authority_cert_t));
+  /* Now copy pointed-to things. */
+  out->cache_info.signed_descriptor_body =
+    tor_strndup(cert->cache_info.signed_descriptor_body,
+                cert->cache_info.signed_descriptor_len);
+  out->cache_info.saved_location = SAVED_NOWHERE;
+  out->identity_key = crypto_pk_dup_key(cert->identity_key);
+  out->signing_key = crypto_pk_dup_key(cert->signing_key);
+
+  return out;
+}
+
+/* =====
+ * Voting
+ * =====*/
+
+/* If <b>opt_value</b> is non-NULL, return "keyword opt_value\n" in a new
+ * string. Otherwise return a new empty string. */
+static char *
+format_line_if_present(const char *keyword, const char *opt_value)
+{
+  if (opt_value) {
+    char *result = NULL;
+    tor_asprintf(&result, "%s %s\n", keyword, opt_value);
+    return result;
+  } else {
+    return tor_strdup("");
+  }
+}
+
+/** Format the recommended/required-relay-client protocols lines for a vote in
+ * a newly allocated string, and return that string. */
+static char *
+format_protocols_lines_for_vote(const networkstatus_t *v3_ns)
+{
+  char *recommended_relay_protocols_line = NULL;
+  char *recommended_client_protocols_line = NULL;
+  char *required_relay_protocols_line = NULL;
+  char *required_client_protocols_line = NULL;
+
+  recommended_relay_protocols_line =
+    format_line_if_present("recommended-relay-protocols",
+                           v3_ns->recommended_relay_protocols);
+  recommended_client_protocols_line =
+    format_line_if_present("recommended-client-protocols",
+                           v3_ns->recommended_client_protocols);
+  required_relay_protocols_line =
+    format_line_if_present("required-relay-protocols",
+                           v3_ns->required_relay_protocols);
+  required_client_protocols_line =
+    format_line_if_present("required-client-protocols",
+                           v3_ns->required_client_protocols);
+
+  char *result = NULL;
+  tor_asprintf(&result, "%s%s%s%s",
+               recommended_relay_protocols_line,
+               recommended_client_protocols_line,
+               required_relay_protocols_line,
+               required_client_protocols_line);
+
+  tor_free(recommended_relay_protocols_line);
+  tor_free(recommended_client_protocols_line);
+  tor_free(required_relay_protocols_line);
+  tor_free(required_client_protocols_line);
+
+  return result;
+}
+
+/** Return a new string containing the string representation of the vote in
+ * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>.
+ * For v3 authorities. */
+STATIC char *
+format_networkstatus_vote(crypto_pk_t *private_signing_key,
+                          networkstatus_t *v3_ns)
+{
+  smartlist_t *chunks = smartlist_new();
+  char *packages = NULL;
+  char fingerprint[FINGERPRINT_LEN+1];
+  char digest[DIGEST_LEN];
+  uint32_t addr;
+  char *protocols_lines = NULL;
+  char *client_versions_line = NULL, *server_versions_line = NULL;
+  char *shared_random_vote_str = NULL;
+  networkstatus_voter_info_t *voter;
+  char *status = NULL;
+
+  tor_assert(private_signing_key);
+  tor_assert(v3_ns->type == NS_TYPE_VOTE || v3_ns->type == NS_TYPE_OPINION);
+
+  voter = smartlist_get(v3_ns->voters, 0);
+
+  addr = voter->addr;
+
+  base16_encode(fingerprint, sizeof(fingerprint),
+                v3_ns->cert->cache_info.identity_digest, DIGEST_LEN);
+
+  client_versions_line = format_line_if_present("client-versions",
+                                                v3_ns->client_versions);
+  server_versions_line = format_line_if_present("server-versions",
+                                                v3_ns->server_versions);
+  protocols_lines = format_protocols_lines_for_vote(v3_ns);
+
+  if (v3_ns->package_lines) {
+    smartlist_t *tmp = smartlist_new();
+    SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p,
+                      if (validate_recommended_package_line(p))
+                        smartlist_add_asprintf(tmp, "package %s\n", p));
+    smartlist_sort_strings(tmp);
+    packages = smartlist_join_strings(tmp, "", 0, NULL);
+    SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp));
+    smartlist_free(tmp);
+  } else {
+    packages = tor_strdup("");
+  }
+
+    /* Get shared random commitments/reveals line(s). */
+  shared_random_vote_str = sr_get_string_for_vote();
+
+  {
+    char published[ISO_TIME_LEN+1];
+    char va[ISO_TIME_LEN+1];
+    char fu[ISO_TIME_LEN+1];
+    char vu[ISO_TIME_LEN+1];
+    char *flags = smartlist_join_strings(v3_ns->known_flags, " ", 0, NULL);
+    /* XXXX Abstraction violation: should be pulling a field out of v3_ns.*/
+    char *flag_thresholds = dirserv_get_flag_thresholds_line();
+    char *params;
+    authority_cert_t *cert = v3_ns->cert;
+    char *methods =
+      make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD,
+                                 MAX_SUPPORTED_CONSENSUS_METHOD, " ");
+    format_iso_time(published, v3_ns->published);
+    format_iso_time(va, v3_ns->valid_after);
+    format_iso_time(fu, v3_ns->fresh_until);
+    format_iso_time(vu, v3_ns->valid_until);
+
+    if (v3_ns->net_params)
+      params = smartlist_join_strings(v3_ns->net_params, " ", 0, NULL);
+    else
+      params = tor_strdup("");
+
+    tor_assert(cert);
+    smartlist_add_asprintf(chunks,
+                 "network-status-version 3\n"
+                 "vote-status %s\n"
+                 "consensus-methods %s\n"
+                 "published %s\n"
+                 "valid-after %s\n"
+                 "fresh-until %s\n"
+                 "valid-until %s\n"
+                 "voting-delay %d %d\n"
+                 "%s%s" /* versions */
+                 "%s" /* protocols */
+                 "%s" /* packages */
+                 "known-flags %s\n"
+                 "flag-thresholds %s\n"
+                 "params %s\n"
+                 "dir-source %s %s %s %s %d %d\n"
+                 "contact %s\n"
+                 "%s", /* shared randomness information */
+                 v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion",
+                 methods,
+                 published, va, fu, vu,
+                 v3_ns->vote_seconds, v3_ns->dist_seconds,
+                 client_versions_line,
+                 server_versions_line,
+                 protocols_lines,
+                 packages,
+                 flags,
+                 flag_thresholds,
+                 params,
+                 voter->nickname, fingerprint, voter->address,
+                 fmt_addr32(addr), voter->dir_port, voter->or_port,
+                 voter->contact,
+                 shared_random_vote_str ?
+                           shared_random_vote_str : "");
+
+    tor_free(params);
+    tor_free(flags);
+    tor_free(flag_thresholds);
+    tor_free(methods);
+    tor_free(shared_random_vote_str);
+
+    if (!tor_digest_is_zero(voter->legacy_id_digest)) {
+      char fpbuf[HEX_DIGEST_LEN+1];
+      base16_encode(fpbuf, sizeof(fpbuf), voter->legacy_id_digest, DIGEST_LEN);
+      smartlist_add_asprintf(chunks, "legacy-dir-key %s\n", fpbuf);
+    }
+
+    smartlist_add(chunks, tor_strndup(cert->cache_info.signed_descriptor_body,
+                                      cert->cache_info.signed_descriptor_len));
+  }
+
+  SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *,
+                          vrs) {
+    char *rsf;
+    vote_microdesc_hash_t *h;
+    rsf = routerstatus_format_entry(&vrs->status,
+                                    vrs->version, vrs->protocols,
+                                    NS_V3_VOTE,
+                                    ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
+                                    vrs);
+    if (rsf)
+      smartlist_add(chunks, rsf);
+
+    for (h = vrs->microdesc; h; h = h->next) {
+      smartlist_add_strdup(chunks, h->microdesc_hash_line);
+    }
+  } SMARTLIST_FOREACH_END(vrs);
+
+  smartlist_add_strdup(chunks, "directory-footer\n");
+
+  /* The digest includes everything up through the space after
+   * directory-signature.  (Yuck.) */
+  crypto_digest_smartlist(digest, DIGEST_LEN, chunks,
+                          "directory-signature ", DIGEST_SHA1);
+
+  {
+    char signing_key_fingerprint[FINGERPRINT_LEN+1];
+    if (crypto_pk_get_fingerprint(private_signing_key,
+                                  signing_key_fingerprint, 0)<0) {
+      log_warn(LD_BUG, "Unable to get fingerprint for signing key");
+      goto err;
+    }
+
+    smartlist_add_asprintf(chunks, "directory-signature %s %s\n", fingerprint,
+                           signing_key_fingerprint);
+  }
+
+  {
+    char *sig = router_get_dirobj_signature(digest, DIGEST_LEN,
+                                            private_signing_key);
+    if (!sig) {
+      log_warn(LD_BUG, "Unable to sign networkstatus vote.");
+      goto err;
+    }
+    smartlist_add(chunks, sig);
+  }
+
+  status = smartlist_join_strings(chunks, "", 0, NULL);
+
+  {
+    networkstatus_t *v;
+    if (!(v = networkstatus_parse_vote_from_string(status, NULL,
+                                                   v3_ns->type))) {
+      log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: "
+              "<<%s>>",
+              v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", status);
+      goto err;
+    }
+    networkstatus_vote_free(v);
+  }
+
+  goto done;
+
+ err:
+  tor_free(status);
+ done:
+  tor_free(client_versions_line);
+  tor_free(server_versions_line);
+  tor_free(protocols_lines);
+  tor_free(packages);
+
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+  return status;
+}
+
+/** Set *<b>timing_out</b> to the intervals at which we would like to vote.
+ * Note that these aren't the intervals we'll use to vote; they're the ones
+ * that we'll vote to use. */
+static void
+dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out)
+{
+  const or_options_t *options = get_options();
+
+  tor_assert(timing_out);
+
+  timing_out->vote_interval = options->V3AuthVotingInterval;
+  timing_out->n_intervals_valid = options->V3AuthNIntervalsValid;
+  timing_out->vote_delay = options->V3AuthVoteDelay;
+  timing_out->dist_delay = options->V3AuthDistDelay;
+}
+
+/* =====
+ * Consensus generation
+ * ===== */
+
+/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with
+ * the digest algorithm <b>alg</b>, decode it and copy it into
+ * <b>digest256_out</b> and return 0.  Otherwise return -1. */
+static int
+vote_routerstatus_find_microdesc_hash(char *digest256_out,
+                                      const vote_routerstatus_t *vrs,
+                                      int method,
+                                      digest_algorithm_t alg)
+{
+  /* XXXX only returns the sha256 method. */
+  const vote_microdesc_hash_t *h;
+  char mstr[64];
+  size_t mlen;
+  char dstr[64];
+
+  tor_snprintf(mstr, sizeof(mstr), "%d", method);
+  mlen = strlen(mstr);
+  tor_snprintf(dstr, sizeof(dstr), " %s=",
+               crypto_digest_algorithm_get_name(alg));
+
+  for (h = vrs->microdesc; h; h = h->next) {
+    const char *cp = h->microdesc_hash_line;
+    size_t num_len;
+    /* cp looks like \d+(,\d+)* (digesttype=val )+ .  Let's hunt for mstr in
+     * the first part. */
+    while (1) {
+      num_len = strspn(cp, "1234567890");
+      if (num_len == mlen && fast_memeq(mstr, cp, mlen)) {
+        /* This is the line. */
+        char buf[BASE64_DIGEST256_LEN+1];
+        /* XXXX ignores extraneous stuff if the digest is too long.  This
+         * seems harmless enough, right? */
+        cp = strstr(cp, dstr);
+        if (!cp)
+          return -1;
+        cp += strlen(dstr);
+        strlcpy(buf, cp, sizeof(buf));
+        return digest256_from_base64(digest256_out, buf);
+      }
+      if (num_len == 0 || cp[num_len] != ',')
+        break;
+      cp += num_len + 1;
+    }
+  }
+  return -1;
+}
+
+/** Given a vote <b>vote</b> (not a consensus!), return its associated
+ * networkstatus_voter_info_t. */
+static networkstatus_voter_info_t *
+get_voter(const networkstatus_t *vote)
+{
+  tor_assert(vote);
+  tor_assert(vote->type == NS_TYPE_VOTE);
+  tor_assert(vote->voters);
+  tor_assert(smartlist_len(vote->voters) == 1);
+  return smartlist_get(vote->voters, 0);
+}
+
+/** Temporary structure used in constructing a list of dir-source entries
+ * for a consensus.  One of these is generated for every vote, and one more
+ * for every legacy key in each vote. */
+typedef struct dir_src_ent_t {
+  networkstatus_t *v;
+  const char *digest;
+  int is_legacy;
+} dir_src_ent_t;
+
+/** Helper for sorting networkstatus_t votes (not consensuses) by the
+ * hash of their voters' identity digests. */
+static int
+compare_votes_by_authority_id_(const void **_a, const void **_b)
+{
+  const networkstatus_t *a = *_a, *b = *_b;
+  return fast_memcmp(get_voter(a)->identity_digest,
+                get_voter(b)->identity_digest, DIGEST_LEN);
+}
+
+/** Helper: Compare the dir_src_ent_ts in *<b>_a</b> and *<b>_b</b> by
+ * their identity digests, and return -1, 0, or 1 depending on their
+ * ordering */
+static int
+compare_dir_src_ents_by_authority_id_(const void **_a, const void **_b)
+{
+  const dir_src_ent_t *a = *_a, *b = *_b;
+  const networkstatus_voter_info_t *a_v = get_voter(a->v),
+    *b_v = get_voter(b->v);
+  const char *a_id, *b_id;
+  a_id = a->is_legacy ? a_v->legacy_id_digest : a_v->identity_digest;
+  b_id = b->is_legacy ? b_v->legacy_id_digest : b_v->identity_digest;
+
+  return fast_memcmp(a_id, b_id, DIGEST_LEN);
+}
+
+/** Given a sorted list of strings <b>in</b>, add every member to <b>out</b>
+ * that occurs more than <b>min</b> times. */
+static void
+get_frequent_members(smartlist_t *out, smartlist_t *in, int min)
+{
+  char *cur = NULL;
+  int count = 0;
+  SMARTLIST_FOREACH_BEGIN(in, char *, cp) {
+    if (cur && !strcmp(cp, cur)) {
+      ++count;
+    } else {
+      if (count > min)
+        smartlist_add(out, cur);
+      cur = cp;
+      count = 1;
+    }
+  } SMARTLIST_FOREACH_END(cp);
+  if (count > min)
+    smartlist_add(out, cur);
+}
+
+/** Given a sorted list of strings <b>lst</b>, return the member that appears
+ * most.  Break ties in favor of later-occurring members. */
+#define get_most_frequent_member(lst)           \
+  smartlist_get_most_frequent_string(lst)
+
+/** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses
+ * that come from the same routerinfo, with the same derived elements.
+ */
+static int
+compare_vote_rs(const vote_routerstatus_t *a, const vote_routerstatus_t *b)
+{
+  int r;
+  tor_assert(a);
+  tor_assert(b);
+
+  if ((r = fast_memcmp(a->status.identity_digest, b->status.identity_digest,
+                  DIGEST_LEN)))
+    return r;
+  if ((r = fast_memcmp(a->status.descriptor_digest,
+                       b->status.descriptor_digest,
+                       DIGEST_LEN)))
+    return r;
+  /* If we actually reached this point, then the identities and
+   * the descriptor digests matched, so somebody is making SHA1 collisions.
+   */
+#define CMP_FIELD(utype, itype, field) do {                             \
+    utype aval = (utype) (itype) a->status.field;                       \
+    utype bval = (utype) (itype) b->status.field;                       \
+    utype u = bval - aval;                                              \
+    itype r2 = (itype) u;                                               \
+    if (r2 < 0) {                                                       \
+      return -1;                                                        \
+    } else if (r2 > 0) {                                                \
+      return 1;                                                         \
+    }                                                                   \
+  } while (0)
+
+  CMP_FIELD(uint64_t, int64_t, published_on);
+
+  if ((r = strcmp(b->status.nickname, a->status.nickname)))
+    return r;
+
+  CMP_FIELD(unsigned, int, addr);
+  CMP_FIELD(unsigned, int, or_port);
+  CMP_FIELD(unsigned, int, dir_port);
+
+  return 0;
+}
+
+/** Helper for sorting routerlists based on compare_vote_rs. */
+static int
+compare_vote_rs_(const void **_a, const void **_b)
+{
+  const vote_routerstatus_t *a = *_a, *b = *_b;
+  return compare_vote_rs(a,b);
+}
+
+/** Helper for sorting OR ports. */
+static int
+compare_orports_(const void **_a, const void **_b)
+{
+  const tor_addr_port_t *a = *_a, *b = *_b;
+  int r;
+
+  if ((r = tor_addr_compare(&a->addr, &b->addr, CMP_EXACT)))
+    return r;
+  if ((r = (((int) b->port) - ((int) a->port))))
+    return r;
+
+  return 0;
+}
+
+/** Given a list of vote_routerstatus_t, all for the same router identity,
+ * return whichever is most frequent, breaking ties in favor of more
+ * recently published vote_routerstatus_t and in case of ties there,
+ * in favor of smaller descriptor digest.
+ */
+static vote_routerstatus_t *
+compute_routerstatus_consensus(smartlist_t *votes, int consensus_method,
+                               char *microdesc_digest256_out,
+                               tor_addr_port_t *best_alt_orport_out)
+{
+  vote_routerstatus_t *most = NULL, *cur = NULL;
+  int most_n = 0, cur_n = 0;
+  time_t most_published = 0;
+
+  /* compare_vote_rs_() sorts the items by identity digest (all the same),
+   * then by SD digest.  That way, if we have a tie that the published_on
+   * date cannot break, we use the descriptor with the smaller digest.
+   */
+  smartlist_sort(votes, compare_vote_rs_);
+  SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) {
+    if (cur && !compare_vote_rs(cur, rs)) {
+      ++cur_n;
+    } else {
+      if (cur && (cur_n > most_n ||
+                  (cur_n == most_n &&
+                   cur->status.published_on > most_published))) {
+        most = cur;
+        most_n = cur_n;
+        most_published = cur->status.published_on;
+      }
+      cur_n = 1;
+      cur = rs;
+    }
+  } SMARTLIST_FOREACH_END(rs);
+
+  if (cur_n > most_n ||
+      (cur && cur_n == most_n && cur->status.published_on > most_published)) {
+    most = cur;
+    // most_n = cur_n; // unused after this point.
+    // most_published = cur->status.published_on; // unused after this point.
+  }
+
+  tor_assert(most);
+
+  /* Vote on potential alternative (sets of) OR port(s) in the winning
+   * routerstatuses.
+   *
+   * XXX prop186 There's at most one alternative OR port (_the_ IPv6
+   * port) for now. */
+  if (best_alt_orport_out) {
+    smartlist_t *alt_orports = smartlist_new();
+    const tor_addr_port_t *most_alt_orport = NULL;
+
+    SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) {
+      tor_assert(rs);
+      if (compare_vote_rs(most, rs) == 0 &&
+          !tor_addr_is_null(&rs->status.ipv6_addr)
+          && rs->status.ipv6_orport) {
+        smartlist_add(alt_orports, tor_addr_port_new(&rs->status.ipv6_addr,
+                                                     rs->status.ipv6_orport));
+      }
+    } SMARTLIST_FOREACH_END(rs);
+
+    smartlist_sort(alt_orports, compare_orports_);
+    most_alt_orport = smartlist_get_most_frequent(alt_orports,
+                                                  compare_orports_);
+    if (most_alt_orport) {
+      memcpy(best_alt_orport_out, most_alt_orport, sizeof(tor_addr_port_t));
+      log_debug(LD_DIR, "\"a\" line winner for %s is %s",
+                most->status.nickname,
+                fmt_addrport(&most_alt_orport->addr, most_alt_orport->port));
+    }
+
+    SMARTLIST_FOREACH(alt_orports, tor_addr_port_t *, ap, tor_free(ap));
+    smartlist_free(alt_orports);
+  }
+
+  if (microdesc_digest256_out) {
+    smartlist_t *digests = smartlist_new();
+    const uint8_t *best_microdesc_digest;
+    SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) {
+        char d[DIGEST256_LEN];
+        if (compare_vote_rs(rs, most))
+          continue;
+        if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method,
+                                                   DIGEST_SHA256))
+          smartlist_add(digests, tor_memdup(d, sizeof(d)));
+    } SMARTLIST_FOREACH_END(rs);
+    smartlist_sort_digests256(digests);
+    best_microdesc_digest = smartlist_get_most_frequent_digest256(digests);
+    if (best_microdesc_digest)
+      memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN);
+    SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp));
+    smartlist_free(digests);
+  }
+
+  return most;
+}
+
+/** Sorting helper: compare two strings based on their values as base-ten
+ * positive integers. (Non-integers are treated as prior to all integers, and
+ * compared lexically.) */
+static int
+cmp_int_strings_(const void **_a, const void **_b)
+{
+  const char *a = *_a, *b = *_b;
+  int ai = (int)tor_parse_long(a, 10, 1, INT_MAX, NULL, NULL);
+  int bi = (int)tor_parse_long(b, 10, 1, INT_MAX, NULL, NULL);
+  if (ai<bi) {
+    return -1;
+  } else if (ai==bi) {
+    if (ai == 0) /* Parsing failed. */
+      return strcmp(a, b);
+    return 0;
+  } else {
+    return 1;
+  }
+}
+
+/** Given a list of networkstatus_t votes, determine and return the number of
+ * the highest consensus method that is supported by 2/3 of the voters. */
+static int
+compute_consensus_method(smartlist_t *votes)
+{
+  smartlist_t *all_methods = smartlist_new();
+  smartlist_t *acceptable_methods = smartlist_new();
+  smartlist_t *tmp = smartlist_new();
+  int min = (smartlist_len(votes) * 2) / 3;
+  int n_ok;
+  int result;
+  SMARTLIST_FOREACH(votes, networkstatus_t *, vote,
+  {
+    tor_assert(vote->supported_methods);
+    smartlist_add_all(tmp, vote->supported_methods);
+    smartlist_sort(tmp, cmp_int_strings_);
+    smartlist_uniq(tmp, cmp_int_strings_, NULL);
+    smartlist_add_all(all_methods, tmp);
+    smartlist_clear(tmp);
+  });
+
+  smartlist_sort(all_methods, cmp_int_strings_);
+  get_frequent_members(acceptable_methods, all_methods, min);
+  n_ok = smartlist_len(acceptable_methods);
+  if (n_ok) {
+    const char *best = smartlist_get(acceptable_methods, n_ok-1);
+    result = (int)tor_parse_long(best, 10, 1, INT_MAX, NULL, NULL);
+  } else {
+    result = 1;
+  }
+  smartlist_free(tmp);
+  smartlist_free(all_methods);
+  smartlist_free(acceptable_methods);
+  return result;
+}
+
+/** Return true iff <b>method</b> is a consensus method that we support. */
+static int
+consensus_method_is_supported(int method)
+{
+  return (method >= MIN_SUPPORTED_CONSENSUS_METHOD) &&
+    (method <= MAX_SUPPORTED_CONSENSUS_METHOD);
+}
+
+/** Return a newly allocated string holding the numbers between low and high
+ * (inclusive) that are supported consensus methods. */
+STATIC char *
+make_consensus_method_list(int low, int high, const char *separator)
+{
+  char *list;
+
+  int i;
+  smartlist_t *lst;
+  lst = smartlist_new();
+  for (i = low; i <= high; ++i) {
+    if (!consensus_method_is_supported(i))
+      continue;
+    smartlist_add_asprintf(lst, "%d", i);
+  }
+  list = smartlist_join_strings(lst, separator, 0, NULL);
+  tor_assert(list);
+  SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp));
+  smartlist_free(lst);
+  return list;
+}
+
+/** Helper: given <b>lst</b>, a list of version strings such that every
+ * version appears once for every versioning voter who recommends it, return a
+ * newly allocated string holding the resulting client-versions or
+ * server-versions list. May change contents of <b>lst</b> */
+static char *
+compute_consensus_versions_list(smartlist_t *lst, int n_versioning)
+{
+  int min = n_versioning / 2;
+  smartlist_t *good = smartlist_new();
+  char *result;
+  sort_version_list(lst, 0);
+  get_frequent_members(good, lst, min);
+  result = smartlist_join_strings(good, ",", 0, NULL);
+  smartlist_free(good);
+  return result;
+}
+
+/** Given a list of K=V values, return the int32_t value corresponding to
+ * KEYWORD=, or default_val if no such value exists, or if the value is
+ * corrupt.
+ */
+STATIC int32_t
+dirvote_get_intermediate_param_value(const smartlist_t *param_list,
+                                     const char *keyword,
+                                     int32_t default_val)
+{
+  unsigned int n_found = 0;
+  int32_t value = default_val;
+
+  SMARTLIST_FOREACH_BEGIN(param_list, const char *, k_v_pair) {
+    if (!strcmpstart(k_v_pair, keyword) && k_v_pair[strlen(keyword)] == '=') {
+      const char *integer_str = &k_v_pair[strlen(keyword)+1];
+      int ok;
+      value = (int32_t)
+        tor_parse_long(integer_str, 10, INT32_MIN, INT32_MAX, &ok, NULL);
+      if (BUG(! ok))
+        return default_val;
+      ++n_found;
+    }
+  } SMARTLIST_FOREACH_END(k_v_pair);
+
+  if (n_found == 1) {
+    return value;
+  } else {
+    tor_assert_nonfatal(n_found == 0);
+    return default_val;
+  }
+}
+
+/** Minimum number of directory authorities voting for a parameter to
+ * include it in the consensus, if consensus method 12 or later is to be
+ * used. See proposal 178 for details. */
+#define MIN_VOTES_FOR_PARAM 3
+
+/** Helper: given a list of valid networkstatus_t, return a new smartlist
+ * containing the contents of the consensus network parameter set.
+ */
+STATIC smartlist_t *
+dirvote_compute_params(smartlist_t *votes, int method, int total_authorities)
+{
+  int i;
+  int32_t *vals;
+
+  int cur_param_len;
+  const char *cur_param;
+  const char *eq;
+
+  const int n_votes = smartlist_len(votes);
+  smartlist_t *output;
+  smartlist_t *param_list = smartlist_new();
+  (void) method;
+
+  /* We require that the parameter lists in the votes are well-formed: that
+     is, that their keywords are unique and sorted, and that their values are
+     between INT32_MIN and INT32_MAX inclusive.  This should be guaranteed by
+     the parsing code. */
+
+  vals = tor_calloc(n_votes, sizeof(int));
+
+  SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+    if (!v->net_params)
+      continue;
+    smartlist_add_all(param_list, v->net_params);
+  } SMARTLIST_FOREACH_END(v);
+
+  if (smartlist_len(param_list) == 0) {
+    tor_free(vals);
+    return param_list;
+  }
+
+  smartlist_sort_strings(param_list);
+  i = 0;
+  cur_param = smartlist_get(param_list, 0);
+  eq = strchr(cur_param, '=');
+  tor_assert(eq);
+  cur_param_len = (int)(eq+1 - cur_param);
+
+  output = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(param_list, const char *, param) {
+    /* resolve spurious clang shallow analysis null pointer errors */
+    tor_assert(param);
+
+    const char *next_param;
+    int ok=0;
+    eq = strchr(param, '=');
+    tor_assert(i<n_votes); /* Make sure we prevented vote-stuffing. */
+    vals[i++] = (int32_t)
+      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
+    tor_assert(ok); /* Already checked these when parsing. */
+
+    if (param_sl_idx+1 == smartlist_len(param_list))
+      next_param = NULL;
+    else
+      next_param = smartlist_get(param_list, param_sl_idx+1);
+
+    if (!next_param || strncmp(next_param, param, cur_param_len)) {
+      /* We've reached the end of a series. */
+      /* Make sure enough authorities voted on this param, unless the
+       * the consensus method we use is too old for that. */
+      if (i > total_authorities/2 ||
+          i >= MIN_VOTES_FOR_PARAM) {
+        int32_t median = median_int32(vals, i);
+        char *out_string = tor_malloc(64+cur_param_len);
+        memcpy(out_string, param, cur_param_len);
+        tor_snprintf(out_string+cur_param_len,64, "%ld", (long)median);
+        smartlist_add(output, out_string);
+      }
+
+      i = 0;
+      if (next_param) {
+        eq = strchr(next_param, '=');
+        cur_param_len = (int)(eq+1 - next_param);
+      }
+    }
+  } SMARTLIST_FOREACH_END(param);
+
+  smartlist_free(param_list);
+  tor_free(vals);
+  return output;
+}
+
+#define RANGE_CHECK(a,b,c,d,e,f,g,mx) \
+       ((a) >= 0 && (a) <= (mx) && (b) >= 0 && (b) <= (mx) && \
+        (c) >= 0 && (c) <= (mx) && (d) >= 0 && (d) <= (mx) && \
+        (e) >= 0 && (e) <= (mx) && (f) >= 0 && (f) <= (mx) && \
+        (g) >= 0 && (g) <= (mx))
+
+#define CHECK_EQ(a, b, margin) \
+     ((a)-(b) >= 0 ? (a)-(b) <= (margin) : (b)-(a) <= (margin))
+
+typedef enum {
+ BW_WEIGHTS_NO_ERROR = 0,
+ BW_WEIGHTS_RANGE_ERROR = 1,
+ BW_WEIGHTS_SUMG_ERROR = 2,
+ BW_WEIGHTS_SUME_ERROR = 3,
+ BW_WEIGHTS_SUMD_ERROR = 4,
+ BW_WEIGHTS_BALANCE_MID_ERROR = 5,
+ BW_WEIGHTS_BALANCE_EG_ERROR = 6
+} bw_weights_error_t;
+
+/**
+ * Verify that any weightings satisfy the balanced formulas.
+ */
+static bw_weights_error_t
+networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg,
+                            int64_t Wme, int64_t Wmd, int64_t Wee,
+                            int64_t Wed, int64_t scale, int64_t G,
+                            int64_t M, int64_t E, int64_t D, int64_t T,
+                            int64_t margin, int do_balance) {
+  bw_weights_error_t berr = BW_WEIGHTS_NO_ERROR;
+
+  // Wed + Wmd + Wgd == 1
+  if (!CHECK_EQ(Wed + Wmd + Wgd, scale, margin)) {
+    berr = BW_WEIGHTS_SUMD_ERROR;
+    goto out;
+  }
+
+  // Wmg + Wgg == 1
+  if (!CHECK_EQ(Wmg + Wgg, scale, margin)) {
+    berr = BW_WEIGHTS_SUMG_ERROR;
+    goto out;
+  }
+
+  // Wme + Wee == 1
+  if (!CHECK_EQ(Wme + Wee, scale, margin)) {
+    berr = BW_WEIGHTS_SUME_ERROR;
+    goto out;
+  }
+
+  // Verify weights within range 0->1
+  if (!RANGE_CHECK(Wgg, Wgd, Wmg, Wme, Wmd, Wed, Wee, scale)) {
+    berr = BW_WEIGHTS_RANGE_ERROR;
+    goto out;
+  }
+
+  if (do_balance) {
+    // Wgg*G + Wgd*D == Wee*E + Wed*D, already scaled
+    if (!CHECK_EQ(Wgg*G + Wgd*D, Wee*E + Wed*D, (margin*T)/3)) {
+      berr = BW_WEIGHTS_BALANCE_EG_ERROR;
+      goto out;
+    }
+
+    // Wgg*G + Wgd*D == M*scale + Wmd*D + Wme*E + Wmg*G, already scaled
+    if (!CHECK_EQ(Wgg*G + Wgd*D, M*scale + Wmd*D + Wme*E + Wmg*G,
+                (margin*T)/3)) {
+      berr = BW_WEIGHTS_BALANCE_MID_ERROR;
+      goto out;
+    }
+  }
+
+ out:
+  if (berr) {
+    log_info(LD_DIR,
+             "Bw weight mismatch %d. G=%"PRId64" M=%"PRId64
+             " E=%"PRId64" D=%"PRId64" T=%"PRId64
+             " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d"
+             " Wgd=%d Wgg=%d Wme=%d Wmg=%d",
+             berr,
+             (G), (M), (E),
+             (D), (T),
+             (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee,
+             (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg);
+  }
+
+  return berr;
+}
+
+/**
+ * This function computes the bandwidth weights for consensus method 10.
+ *
+ * It returns true if weights could be computed, false otherwise.
+ */
+int
+networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
+                                     int64_t M, int64_t E, int64_t D,
+                                     int64_t T, int64_t weight_scale)
+{
+  bw_weights_error_t berr = 0;
+  int64_t Wgg = -1, Wgd = -1;
+  int64_t Wmg = -1, Wme = -1, Wmd = -1;
+  int64_t Wed = -1, Wee = -1;
+  const char *casename;
+
+  if (G <= 0 || M <= 0 || E <= 0 || D <= 0) {
+    log_warn(LD_DIR, "Consensus with empty bandwidth: "
+                     "G=%"PRId64" M=%"PRId64" E=%"PRId64
+                     " D=%"PRId64" T=%"PRId64,
+             (G), (M), (E),
+             (D), (T));
+    return 0;
+  }
+
+  /*
+   * Computed from cases in 3.8.3 of dir-spec.txt
+   *
+   * 1. Neither are scarce
+   * 2. Both Guard and Exit are scarce
+   *    a. R+D <= S
+   *    b. R+D > S
+   * 3. One of Guard or Exit is scarce
+   *    a. S+D < T/3
+   *    b. S+D >= T/3
+   */
+  if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3
+    /* Case 1: Neither are scarce.  */
+    casename = "Case 1 (Wgd=Wmd=Wed)";
+    Wgd = weight_scale/3;
+    Wed = weight_scale/3;
+    Wmd = weight_scale/3;
+    Wee = (weight_scale*(E+G+M))/(3*E);
+    Wme = weight_scale - Wee;
+    Wmg = (weight_scale*(2*G-E-M))/(3*G);
+    Wgg = weight_scale - Wmg;
+
+    berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed,
+                                       weight_scale, G, M, E, D, T, 10, 1);
+
+    if (berr) {
+      log_warn(LD_DIR,
+             "Bw Weights error %d for %s v10. G=%"PRId64" M=%"PRId64
+             " E=%"PRId64" D=%"PRId64" T=%"PRId64
+             " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d"
+             " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d",
+             berr, casename,
+             (G), (M), (E),
+             (D), (T),
+             (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee,
+             (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale);
+      return 0;
+    }
+  } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3
+    int64_t R = MIN(E, G);
+    int64_t S = MAX(E, G);
+    /*
+     * Case 2: Both Guards and Exits are scarce
+     * Balance D between E and G, depending upon
+     * D capacity and scarcity.
+     */
+    if (R+D < S) { // Subcase a
+      Wgg = weight_scale;
+      Wee = weight_scale;
+      Wmg = 0;
+      Wme = 0;
+      Wmd = 0;
+      if (E < G) {
+        casename = "Case 2a (E scarce)";
+        Wed = weight_scale;
+        Wgd = 0;
+      } else { /* E >= G */
+        casename = "Case 2a (G scarce)";
+        Wed = 0;
+        Wgd = weight_scale;
+      }
+    } else { // Subcase b: R+D >= S
+      casename = "Case 2b1 (Wgg=weight_scale, Wmd=Wgd)";
+      Wee = (weight_scale*(E - G + M))/E;
+      Wed = (weight_scale*(D - 2*E + 4*G - 2*M))/(3*D);
+      Wme = (weight_scale*(G-M))/E;
+      Wmg = 0;
+      Wgg = weight_scale;
+      Wmd = (weight_scale - Wed)/2;
+      Wgd = (weight_scale - Wed)/2;
+
+      berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed,
+                                       weight_scale, G, M, E, D, T, 10, 1);
+
+      if (berr) {
+        casename = "Case 2b2 (Wgg=weight_scale, Wee=weight_scale)";
+        Wgg = weight_scale;
+        Wee = weight_scale;
+        Wed = (weight_scale*(D - 2*E + G + M))/(3*D);
+        Wmd = (weight_scale*(D - 2*M + G + E))/(3*D);
+        Wme = 0;
+        Wmg = 0;
+
+        if (Wmd < 0) { // Can happen if M > T/3
+          casename = "Case 2b3 (Wmd=0)";
+          Wmd = 0;
+          log_warn(LD_DIR,
+                   "Too much Middle bandwidth on the network to calculate "
+                   "balanced bandwidth-weights. Consider increasing the "
+                   "number of Guard nodes by lowering the requirements.");
+        }
+        Wgd = weight_scale - Wed - Wmd;
+        berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee,
+                  Wed, weight_scale, G, M, E, D, T, 10, 1);
+      }
+      if (berr != BW_WEIGHTS_NO_ERROR &&
+              berr != BW_WEIGHTS_BALANCE_MID_ERROR) {
+        log_warn(LD_DIR,
+             "Bw Weights error %d for %s v10. G=%"PRId64" M=%"PRId64
+             " E=%"PRId64" D=%"PRId64" T=%"PRId64
+             " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d"
+             " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d",
+             berr, casename,
+             (G), (M), (E),
+             (D), (T),
+             (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee,
+             (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale);
+        return 0;
+      }
+    }
+  } else { // if (E < T/3 || G < T/3) {
+    int64_t S = MIN(E, G);
+    // Case 3: Exactly one of Guard or Exit is scarce
+    if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) {
+      log_warn(LD_BUG,
+           "Bw-Weights Case 3 v10 but with G=%"PRId64" M="
+           "%"PRId64" E=%"PRId64" D=%"PRId64" T=%"PRId64,
+               (G), (M), (E),
+               (D), (T));
+    }
+
+    if (3*(S+D) < T) { // Subcase a: S+D < T/3
+      if (G < E) {
+        casename = "Case 3a (G scarce)";
+        Wgg = Wgd = weight_scale;
+        Wmd = Wed = Wmg = 0;
+        // Minor subcase, if E is more scarce than M,
+        // keep its bandwidth in place.
+        if (E < M) Wme = 0;
+        else Wme = (weight_scale*(E-M))/(2*E);
+        Wee = weight_scale-Wme;
+      } else { // G >= E
+        casename = "Case 3a (E scarce)";
+        Wee = Wed = weight_scale;
+        Wmd = Wgd = Wme = 0;
+        // Minor subcase, if G is more scarce than M,
+        // keep its bandwidth in place.
+        if (G < M) Wmg = 0;
+        else Wmg = (weight_scale*(G-M))/(2*G);
+        Wgg = weight_scale-Wmg;
+      }
+    } else { // Subcase b: S+D >= T/3
+      // D != 0 because S+D >= T/3
+      if (G < E) {
+        casename = "Case 3bg (G scarce, Wgg=weight_scale, Wmd == Wed)";
+        Wgg = weight_scale;
+        Wgd = (weight_scale*(D - 2*G + E + M))/(3*D);
+        Wmg = 0;
+        Wee = (weight_scale*(E+M))/(2*E);
+        Wme = weight_scale - Wee;
+        Wmd = (weight_scale - Wgd)/2;
+        Wed = (weight_scale - Wgd)/2;
+
+        berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee,
+                    Wed, weight_scale, G, M, E, D, T, 10, 1);
+      } else { // G >= E
+        casename = "Case 3be (E scarce, Wee=weight_scale, Wmd == Wgd)";
+        Wee = weight_scale;
+        Wed = (weight_scale*(D - 2*E + G + M))/(3*D);
+        Wme = 0;
+        Wgg = (weight_scale*(G+M))/(2*G);
+        Wmg = weight_scale - Wgg;
+        Wmd = (weight_scale - Wed)/2;
+        Wgd = (weight_scale - Wed)/2;
+
+        berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee,
+                      Wed, weight_scale, G, M, E, D, T, 10, 1);
+      }
+      if (berr) {
+        log_warn(LD_DIR,
+             "Bw Weights error %d for %s v10. G=%"PRId64" M=%"PRId64
+             " E=%"PRId64" D=%"PRId64" T=%"PRId64
+             " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d"
+             " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d",
+             berr, casename,
+             (G), (M), (E),
+             (D), (T),
+             (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee,
+             (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale);
+        return 0;
+      }
+    }
+  }
+
+  /* We cast down the weights to 32 bit ints on the assumption that
+   * weight_scale is ~= 10000. We need to ensure a rogue authority
+   * doesn't break this assumption to rig our weights */
+  tor_assert(0 < weight_scale && weight_scale <= INT32_MAX);
+
+  /*
+   * Provide Wgm=Wgg, Wmm=weight_scale, Wem=Wee, Weg=Wed. May later determine
+   * that middle nodes need different bandwidth weights for dirport traffic,
+   * or that weird exit policies need special weight, or that bridges
+   * need special weight.
+   *
+   * NOTE: This list is sorted.
+   */
+  smartlist_add_asprintf(chunks,
+     "bandwidth-weights Wbd=%d Wbe=%d Wbg=%d Wbm=%d "
+     "Wdb=%d "
+     "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d "
+     "Wgb=%d Wgd=%d Wgg=%d Wgm=%d "
+     "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n",
+     (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale,
+     (int)weight_scale,
+     (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee,
+     (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg,
+     (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale);
+
+  log_notice(LD_CIRC, "Computed bandwidth weights for %s with v10: "
+             "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+             " T=%"PRId64,
+             casename,
+             (G), (M), (E),
+             (D), (T));
+  return 1;
+}
+
+/** Update total bandwidth weights (G/M/E/D/T) with the bandwidth of
+ *  the router in <b>rs</b>. */
+static void
+update_total_bandwidth_weights(const routerstatus_t *rs,
+                               int is_exit, int is_guard,
+                               int64_t *G, int64_t *M, int64_t *E, int64_t *D,
+                               int64_t *T)
+{
+  int default_bandwidth = rs->bandwidth_kb;
+  int guardfraction_bandwidth = 0;
+
+  if (!rs->has_bandwidth) {
+    log_info(LD_BUG, "Missing consensus bandwidth for router %s",
+             rs->nickname);
+    return;
+  }
+
+  /* If this routerstatus represents a guard that we have
+   * guardfraction information on, use it to calculate its actual
+   * bandwidth. From proposal236:
+   *
+   *    Similarly, when calculating the bandwidth-weights line as in
+   *    section 3.8.3 of dir-spec.txt, directory authorities should treat N
+   *    as if fraction F of its bandwidth has the guard flag and (1-F) does
+   *    not.  So when computing the totals G,M,E,D, each relay N with guard
+   *    visibility fraction F and bandwidth B should be added as follows:
+   *
+   *    G' = G + F*B, if N does not have the exit flag
+   *    M' = M + (1-F)*B, if N does not have the exit flag
+   *
+   *    or
+   *
+   *    D' = D + F*B, if N has the exit flag
+   *    E' = E + (1-F)*B, if N has the exit flag
+   *
+   * In this block of code, we prepare the bandwidth values by setting
+   * the default_bandwidth to F*B and guardfraction_bandwidth to (1-F)*B.
+   */
+  if (rs->has_guardfraction) {
+    guardfraction_bandwidth_t guardfraction_bw;
+
+    tor_assert(is_guard);
+
+    guard_get_guardfraction_bandwidth(&guardfraction_bw,
+                                      rs->bandwidth_kb,
+                                      rs->guardfraction_percentage);
+
+    default_bandwidth = guardfraction_bw.guard_bw;
+    guardfraction_bandwidth = guardfraction_bw.non_guard_bw;
+  }
+
+  /* Now calculate the total bandwidth weights with or without
+   * guardfraction. Depending on the flags of the relay, add its
+   * bandwidth to the appropriate weight pool. If it's a guard and
+   * guardfraction is enabled, add its bandwidth to both pools as
+   * indicated by the previous comment.
+   */
+  *T += default_bandwidth;
+  if (is_exit && is_guard) {
+
+    *D += default_bandwidth;
+    if (rs->has_guardfraction) {
+      *E += guardfraction_bandwidth;
+    }
+
+  } else if (is_exit) {
+
+    *E += default_bandwidth;
+
+  } else if (is_guard) {
+
+    *G += default_bandwidth;
+    if (rs->has_guardfraction) {
+      *M += guardfraction_bandwidth;
+    }
+
+  } else {
+
+    *M += default_bandwidth;
+  }
+}
+
+/** Considering the different recommended/required protocols sets as a
+ * 4-element array, return the element from <b>vote</b> for that protocol
+ * set.
+ */
+static const char *
+get_nth_protocol_set_vote(int n, const networkstatus_t *vote)
+{
+  switch (n) {
+    case 0: return vote->recommended_client_protocols;
+    case 1: return vote->recommended_relay_protocols;
+    case 2: return vote->required_client_protocols;
+    case 3: return vote->required_relay_protocols;
+    default:
+      tor_assert_unreached();
+      return NULL;
+  }
+}
+
+/** Considering the different recommended/required protocols sets as a
+ * 4-element array, return a newly allocated string for the consensus value
+ * for the n'th set.
+ */
+static char *
+compute_nth_protocol_set(int n, int n_voters, const smartlist_t *votes)
+{
+  const char *keyword;
+  smartlist_t *proto_votes = smartlist_new();
+  int threshold;
+  switch (n) {
+    case 0:
+      keyword = "recommended-client-protocols";
+      threshold = CEIL_DIV(n_voters, 2);
+      break;
+    case 1:
+      keyword = "recommended-relay-protocols";
+      threshold = CEIL_DIV(n_voters, 2);
+      break;
+    case 2:
+      keyword = "required-client-protocols";
+      threshold = CEIL_DIV(n_voters * 2, 3);
+      break;
+    case 3:
+      keyword = "required-relay-protocols";
+      threshold = CEIL_DIV(n_voters * 2, 3);
+      break;
+    default:
+      tor_assert_unreached();
+      return NULL;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(votes, const networkstatus_t *, ns) {
+    const char *v = get_nth_protocol_set_vote(n, ns);
+    if (v)
+      smartlist_add(proto_votes, (void*)v);
+  } SMARTLIST_FOREACH_END(ns);
+
+  char *protocols = protover_compute_vote(proto_votes, threshold);
+  smartlist_free(proto_votes);
+
+  char *result = NULL;
+  tor_asprintf(&result, "%s %s\n", keyword, protocols);
+  tor_free(protocols);
+
+  return result;
+}
+
+/** Given a list of vote networkstatus_t in <b>votes</b>, our public
+ * authority <b>identity_key</b>, our private authority <b>signing_key</b>,
+ * and the number of <b>total_authorities</b> that we believe exist in our
+ * voting quorum, generate the text of a new v3 consensus or microdescriptor
+ * consensus (depending on <b>flavor</b>), and return the value in a newly
+ * allocated string.
+ *
+ * Note: this function DOES NOT check whether the votes are from
+ * recognized authorities.   (dirvote_add_vote does that.)
+ *
+ * <strong>WATCH OUT</strong>: You need to think before you change the
+ * behavior of this function, or of the functions it calls! If some
+ * authorities compute the consensus with a different algorithm than
+ * others, they will not reach the same result, and they will not all
+ * sign the same thing!  If you really need to change the algorithm
+ * here, you should allocate a new "consensus_method" for the new
+ * behavior, and make the new behavior conditional on a new-enough
+ * consensus_method.
+ **/
+STATIC char *
+networkstatus_compute_consensus(smartlist_t *votes,
+                                int total_authorities,
+                                crypto_pk_t *identity_key,
+                                crypto_pk_t *signing_key,
+                                const char *legacy_id_key_digest,
+                                crypto_pk_t *legacy_signing_key,
+                                consensus_flavor_t flavor)
+{
+  smartlist_t *chunks;
+  char *result = NULL;
+  int consensus_method;
+  time_t valid_after, fresh_until, valid_until;
+  int vote_seconds, dist_seconds;
+  char *client_versions = NULL, *server_versions = NULL;
+  smartlist_t *flags;
+  const char *flavor_name;
+  uint32_t max_unmeasured_bw_kb = DEFAULT_MAX_UNMEASURED_BW_KB;
+  int64_t G, M, E, D, T; /* For bandwidth weights */
+  const routerstatus_format_type_t rs_format =
+    flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC;
+  char *params = NULL;
+  char *packages = NULL;
+  int added_weights = 0;
+  dircollator_t *collator = NULL;
+  smartlist_t *param_list = NULL;
+
+  tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC);
+  tor_assert(total_authorities >= smartlist_len(votes));
+  tor_assert(total_authorities > 0);
+
+  flavor_name = networkstatus_get_flavor_name(flavor);
+
+  if (!smartlist_len(votes)) {
+    log_warn(LD_DIR, "Can't compute a consensus from no votes.");
+    return NULL;
+  }
+  flags = smartlist_new();
+
+  consensus_method = compute_consensus_method(votes);
+  if (consensus_method_is_supported(consensus_method)) {
+    log_info(LD_DIR, "Generating consensus using method %d.",
+             consensus_method);
+  } else {
+    log_warn(LD_DIR, "The other authorities will use consensus method %d, "
+             "which I don't support.  Maybe I should upgrade!",
+             consensus_method);
+    consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD;
+  }
+
+  if (consensus_method >= MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE) {
+    /* It's smarter to initialize these weights to 1, so that later on,
+     * we can't accidentally divide by zero. */
+    G = M = E = D = 1;
+    T = 4;
+  } else {
+    /* ...but originally, they were set to zero. */
+    G = M = E = D = T = 0;
+  }
+
+  /* Compute medians of time-related things, and figure out how many
+   * routers we might need to talk about. */
+  {
+    int n_votes = smartlist_len(votes);
+    time_t *va_times = tor_calloc(n_votes, sizeof(time_t));
+    time_t *fu_times = tor_calloc(n_votes, sizeof(time_t));
+    time_t *vu_times = tor_calloc(n_votes, sizeof(time_t));
+    int *votesec_list = tor_calloc(n_votes, sizeof(int));
+    int *distsec_list = tor_calloc(n_votes, sizeof(int));
+    int n_versioning_clients = 0, n_versioning_servers = 0;
+    smartlist_t *combined_client_versions = smartlist_new();
+    smartlist_t *combined_server_versions = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+      tor_assert(v->type == NS_TYPE_VOTE);
+      va_times[v_sl_idx] = v->valid_after;
+      fu_times[v_sl_idx] = v->fresh_until;
+      vu_times[v_sl_idx] = v->valid_until;
+      votesec_list[v_sl_idx] = v->vote_seconds;
+      distsec_list[v_sl_idx] = v->dist_seconds;
+      if (v->client_versions) {
+        smartlist_t *cv = smartlist_new();
+        ++n_versioning_clients;
+        smartlist_split_string(cv, v->client_versions, ",",
+                               SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+        sort_version_list(cv, 1);
+        smartlist_add_all(combined_client_versions, cv);
+        smartlist_free(cv); /* elements get freed later. */
+      }
+      if (v->server_versions) {
+        smartlist_t *sv = smartlist_new();
+        ++n_versioning_servers;
+        smartlist_split_string(sv, v->server_versions, ",",
+                               SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+        sort_version_list(sv, 1);
+        smartlist_add_all(combined_server_versions, sv);
+        smartlist_free(sv); /* elements get freed later. */
+      }
+      SMARTLIST_FOREACH(v->known_flags, const char *, cp,
+                        smartlist_add_strdup(flags, cp));
+    } SMARTLIST_FOREACH_END(v);
+    valid_after = median_time(va_times, n_votes);
+    fresh_until = median_time(fu_times, n_votes);
+    valid_until = median_time(vu_times, n_votes);
+    vote_seconds = median_int(votesec_list, n_votes);
+    dist_seconds = median_int(distsec_list, n_votes);
+
+    tor_assert(valid_after +
+               (get_options()->TestingTorNetwork ?
+                MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= fresh_until);
+    tor_assert(fresh_until +
+               (get_options()->TestingTorNetwork ?
+                MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= valid_until);
+    tor_assert(vote_seconds >= MIN_VOTE_SECONDS);
+    tor_assert(dist_seconds >= MIN_DIST_SECONDS);
+
+    server_versions = compute_consensus_versions_list(combined_server_versions,
+                                                      n_versioning_servers);
+    client_versions = compute_consensus_versions_list(combined_client_versions,
+                                                      n_versioning_clients);
+    packages = compute_consensus_package_lines(votes);
+
+    SMARTLIST_FOREACH(combined_server_versions, char *, cp, tor_free(cp));
+    SMARTLIST_FOREACH(combined_client_versions, char *, cp, tor_free(cp));
+    smartlist_free(combined_server_versions);
+    smartlist_free(combined_client_versions);
+
+    smartlist_add_strdup(flags, "NoEdConsensus");
+
+    smartlist_sort_strings(flags);
+    smartlist_uniq_strings(flags);
+
+    tor_free(va_times);
+    tor_free(fu_times);
+    tor_free(vu_times);
+    tor_free(votesec_list);
+    tor_free(distsec_list);
+  }
+
+  chunks = smartlist_new();
+
+  {
+    char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1],
+      vu_buf[ISO_TIME_LEN+1];
+    char *flaglist;
+    format_iso_time(va_buf, valid_after);
+    format_iso_time(fu_buf, fresh_until);
+    format_iso_time(vu_buf, valid_until);
+    flaglist = smartlist_join_strings(flags, " ", 0, NULL);
+
+    smartlist_add_asprintf(chunks, "network-status-version 3%s%s\n"
+                 "vote-status consensus\n",
+                 flavor == FLAV_NS ? "" : " ",
+                 flavor == FLAV_NS ? "" : flavor_name);
+
+    smartlist_add_asprintf(chunks, "consensus-method %d\n",
+                           consensus_method);
+
+    smartlist_add_asprintf(chunks,
+                 "valid-after %s\n"
+                 "fresh-until %s\n"
+                 "valid-until %s\n"
+                 "voting-delay %d %d\n"
+                 "client-versions %s\n"
+                 "server-versions %s\n"
+                 "%s" /* packages */
+                 "known-flags %s\n",
+                 va_buf, fu_buf, vu_buf,
+                 vote_seconds, dist_seconds,
+                 client_versions, server_versions,
+                 packages,
+                 flaglist);
+
+    tor_free(flaglist);
+  }
+
+  {
+    int num_dirauth = get_n_authorities(V3_DIRINFO);
+    int idx;
+    for (idx = 0; idx < 4; ++idx) {
+      char *proto_line = compute_nth_protocol_set(idx, num_dirauth, votes);
+      if (BUG(!proto_line))
+        continue;
+      smartlist_add(chunks, proto_line);
+    }
+  }
+
+  param_list = dirvote_compute_params(votes, consensus_method,
+                                      total_authorities);
+  if (smartlist_len(param_list)) {
+    params = smartlist_join_strings(param_list, " ", 0, NULL);
+    smartlist_add_strdup(chunks, "params ");
+    smartlist_add(chunks, params);
+    smartlist_add_strdup(chunks, "\n");
+  }
+
+  {
+    int num_dirauth = get_n_authorities(V3_DIRINFO);
+    /* Default value of this is 2/3 of the total number of authorities. For
+     * instance, if we have 9 dirauth, the default value is 6. The following
+     * calculation will round it down. */
+    int32_t num_srv_agreements =
+      dirvote_get_intermediate_param_value(param_list,
+                                           "AuthDirNumSRVAgreements",
+                                           (num_dirauth * 2) / 3);
+    /* Add the shared random value. */
+    char *srv_lines = sr_get_string_for_consensus(votes, num_srv_agreements);
+    if (srv_lines != NULL) {
+      smartlist_add(chunks, srv_lines);
+    }
+  }
+
+  /* Sort the votes. */
+  smartlist_sort(votes, compare_votes_by_authority_id_);
+  /* Add the authority sections. */
+  {
+    smartlist_t *dir_sources = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+      dir_src_ent_t *e = tor_malloc_zero(sizeof(dir_src_ent_t));
+      e->v = v;
+      e->digest = get_voter(v)->identity_digest;
+      e->is_legacy = 0;
+      smartlist_add(dir_sources, e);
+      if (!tor_digest_is_zero(get_voter(v)->legacy_id_digest)) {
+        dir_src_ent_t *e_legacy = tor_malloc_zero(sizeof(dir_src_ent_t));
+        e_legacy->v = v;
+        e_legacy->digest = get_voter(v)->legacy_id_digest;
+        e_legacy->is_legacy = 1;
+        smartlist_add(dir_sources, e_legacy);
+      }
+    } SMARTLIST_FOREACH_END(v);
+    smartlist_sort(dir_sources, compare_dir_src_ents_by_authority_id_);
+
+    SMARTLIST_FOREACH_BEGIN(dir_sources, const dir_src_ent_t *, e) {
+      char fingerprint[HEX_DIGEST_LEN+1];
+      char votedigest[HEX_DIGEST_LEN+1];
+      networkstatus_t *v = e->v;
+      networkstatus_voter_info_t *voter = get_voter(v);
+
+      base16_encode(fingerprint, sizeof(fingerprint), e->digest, DIGEST_LEN);
+      base16_encode(votedigest, sizeof(votedigest), voter->vote_digest,
+                    DIGEST_LEN);
+
+      smartlist_add_asprintf(chunks,
+                   "dir-source %s%s %s %s %s %d %d\n",
+                   voter->nickname, e->is_legacy ? "-legacy" : "",
+                   fingerprint, voter->address, fmt_addr32(voter->addr),
+                   voter->dir_port,
+                   voter->or_port);
+      if (! e->is_legacy) {
+        smartlist_add_asprintf(chunks,
+                     "contact %s\n"
+                     "vote-digest %s\n",
+                     voter->contact,
+                     votedigest);
+      }
+    } SMARTLIST_FOREACH_END(e);
+    SMARTLIST_FOREACH(dir_sources, dir_src_ent_t *, e, tor_free(e));
+    smartlist_free(dir_sources);
+  }
+
+  {
+    char *max_unmeasured_param = NULL;
+    /* XXXX Extract this code into a common function.  Or don't!  see #19011 */
+    if (params) {
+      if (strcmpstart(params, "maxunmeasuredbw=") == 0)
+        max_unmeasured_param = params;
+      else
+        max_unmeasured_param = strstr(params, " maxunmeasuredbw=");
+    }
+    if (max_unmeasured_param) {
+      int ok = 0;
+      char *eq = strchr(max_unmeasured_param, '=');
+      if (eq) {
+        max_unmeasured_bw_kb = (uint32_t)
+          tor_parse_ulong(eq+1, 10, 1, UINT32_MAX, &ok, NULL);
+        if (!ok) {
+          log_warn(LD_DIR, "Bad element '%s' in max unmeasured bw param",
+                   escaped(max_unmeasured_param));
+          max_unmeasured_bw_kb = DEFAULT_MAX_UNMEASURED_BW_KB;
+        }
+      }
+    }
+  }
+
+  /* Add the actual router entries. */
+  {
+    int *size; /* size[j] is the number of routerstatuses in votes[j]. */
+    int *flag_counts; /* The number of voters that list flag[j] for the
+                       * currently considered router. */
+    int i;
+    smartlist_t *matching_descs = smartlist_new();
+    smartlist_t *chosen_flags = smartlist_new();
+    smartlist_t *versions = smartlist_new();
+    smartlist_t *protocols = smartlist_new();
+    smartlist_t *exitsummaries = smartlist_new();
+    uint32_t *bandwidths_kb = tor_calloc(smartlist_len(votes),
+                                         sizeof(uint32_t));
+    uint32_t *measured_bws_kb = tor_calloc(smartlist_len(votes),
+                                           sizeof(uint32_t));
+    uint32_t *measured_guardfraction = tor_calloc(smartlist_len(votes),
+                                                  sizeof(uint32_t));
+    int num_bandwidths;
+    int num_mbws;
+    int num_guardfraction_inputs;
+
+    int *n_voter_flags; /* n_voter_flags[j] is the number of flags that
+                         * votes[j] knows about. */
+    int *n_flag_voters; /* n_flag_voters[f] is the number of votes that care
+                         * about flags[f]. */
+    int **flag_map; /* flag_map[j][b] is an index f such that flag_map[f]
+                     * is the same flag as votes[j]->known_flags[b]. */
+    int *named_flag; /* Index of the flag "Named" for votes[j] */
+    int *unnamed_flag; /* Index of the flag "Unnamed" for votes[j] */
+    int n_authorities_measuring_bandwidth;
+
+    strmap_t *name_to_id_map = strmap_new();
+    char conflict[DIGEST_LEN];
+    char unknown[DIGEST_LEN];
+    memset(conflict, 0, sizeof(conflict));
+    memset(unknown, 0xff, sizeof(conflict));
+
+    size = tor_calloc(smartlist_len(votes), sizeof(int));
+    n_voter_flags = tor_calloc(smartlist_len(votes), sizeof(int));
+    n_flag_voters = tor_calloc(smartlist_len(flags), sizeof(int));
+    flag_map = tor_calloc(smartlist_len(votes), sizeof(int *));
+    named_flag = tor_calloc(smartlist_len(votes), sizeof(int));
+    unnamed_flag = tor_calloc(smartlist_len(votes), sizeof(int));
+    for (i = 0; i < smartlist_len(votes); ++i)
+      unnamed_flag[i] = named_flag[i] = -1;
+
+    /* Build the flag indexes. Note that no vote can have more than 64 members
+     * for known_flags, so no value will be greater than 63, so it's safe to
+     * do UINT64_C(1) << index on these values.  But note also that
+     * named_flag and unnamed_flag are initialized to -1, so we need to check
+     * that they're actually set before doing UINT64_C(1) << index with
+     * them.*/
+    SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+      flag_map[v_sl_idx] = tor_calloc(smartlist_len(v->known_flags),
+                                      sizeof(int));
+      if (smartlist_len(v->known_flags) > MAX_KNOWN_FLAGS_IN_VOTE) {
+        log_warn(LD_BUG, "Somehow, a vote has %d entries in known_flags",
+                 smartlist_len(v->known_flags));
+      }
+      SMARTLIST_FOREACH_BEGIN(v->known_flags, const char *, fl) {
+        int p = smartlist_string_pos(flags, fl);
+        tor_assert(p >= 0);
+        flag_map[v_sl_idx][fl_sl_idx] = p;
+        ++n_flag_voters[p];
+        if (!strcmp(fl, "Named"))
+          named_flag[v_sl_idx] = fl_sl_idx;
+        if (!strcmp(fl, "Unnamed"))
+          unnamed_flag[v_sl_idx] = fl_sl_idx;
+      } SMARTLIST_FOREACH_END(fl);
+      n_voter_flags[v_sl_idx] = smartlist_len(v->known_flags);
+      size[v_sl_idx] = smartlist_len(v->routerstatus_list);
+    } SMARTLIST_FOREACH_END(v);
+
+    /* Named and Unnamed get treated specially */
+    {
+      SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+        uint64_t nf;
+        if (named_flag[v_sl_idx]<0)
+          continue;
+        nf = UINT64_C(1) << named_flag[v_sl_idx];
+        SMARTLIST_FOREACH_BEGIN(v->routerstatus_list,
+                                vote_routerstatus_t *, rs) {
+
+          if ((rs->flags & nf) != 0) {
+            const char *d = strmap_get_lc(name_to_id_map, rs->status.nickname);
+            if (!d) {
+              /* We have no name officially mapped to this digest. */
+              strmap_set_lc(name_to_id_map, rs->status.nickname,
+                            rs->status.identity_digest);
+            } else if (d != conflict &&
+                fast_memcmp(d, rs->status.identity_digest, DIGEST_LEN)) {
+              /* Authorities disagree about this nickname. */
+              strmap_set_lc(name_to_id_map, rs->status.nickname, conflict);
+            } else {
+              /* It's already a conflict, or it's already this ID. */
+            }
+          }
+        } SMARTLIST_FOREACH_END(rs);
+      } SMARTLIST_FOREACH_END(v);
+
+      SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+        uint64_t uf;
+        if (unnamed_flag[v_sl_idx]<0)
+          continue;
+        uf = UINT64_C(1) << unnamed_flag[v_sl_idx];
+        SMARTLIST_FOREACH_BEGIN(v->routerstatus_list,
+                                vote_routerstatus_t *, rs) {
+          if ((rs->flags & uf) != 0) {
+            const char *d = strmap_get_lc(name_to_id_map, rs->status.nickname);
+            if (d == conflict || d == unknown) {
+              /* Leave it alone; we know what it is. */
+            } else if (!d) {
+              /* We have no name officially mapped to this digest. */
+              strmap_set_lc(name_to_id_map, rs->status.nickname, unknown);
+            } else if (fast_memeq(d, rs->status.identity_digest, DIGEST_LEN)) {
+              /* Authorities disagree about this nickname. */
+              strmap_set_lc(name_to_id_map, rs->status.nickname, conflict);
+            } else {
+              /* It's mapped to a different name. */
+            }
+          }
+        } SMARTLIST_FOREACH_END(rs);
+      } SMARTLIST_FOREACH_END(v);
+    }
+
+    /* We need to know how many votes measure bandwidth. */
+    n_authorities_measuring_bandwidth = 0;
+    SMARTLIST_FOREACH(votes, const networkstatus_t *, v,
+       if (v->has_measured_bws) {
+         ++n_authorities_measuring_bandwidth;
+       }
+    );
+
+    /* Populate the collator */
+    collator = dircollator_new(smartlist_len(votes), total_authorities);
+    SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+      dircollator_add_vote(collator, v);
+    } SMARTLIST_FOREACH_END(v);
+
+    dircollator_collate(collator, consensus_method);
+
+    /* Now go through all the votes */
+    flag_counts = tor_calloc(smartlist_len(flags), sizeof(int));
+    const int num_routers = dircollator_n_routers(collator);
+    for (i = 0; i < num_routers; ++i) {
+      vote_routerstatus_t **vrs_lst =
+        dircollator_get_votes_for_router(collator, i);
+
+      vote_routerstatus_t *rs;
+      routerstatus_t rs_out;
+      const char *current_rsa_id = NULL;
+      const char *chosen_version;
+      const char *chosen_protocol_list;
+      const char *chosen_name = NULL;
+      int exitsummary_disagreement = 0;
+      int is_named = 0, is_unnamed = 0, is_running = 0, is_valid = 0;
+      int is_guard = 0, is_exit = 0, is_bad_exit = 0;
+      int naming_conflict = 0;
+      int n_listing = 0;
+      char microdesc_digest[DIGEST256_LEN];
+      tor_addr_port_t alt_orport = {TOR_ADDR_NULL, 0};
+
+      memset(flag_counts, 0, sizeof(int)*smartlist_len(flags));
+      smartlist_clear(matching_descs);
+      smartlist_clear(chosen_flags);
+      smartlist_clear(versions);
+      smartlist_clear(protocols);
+      num_bandwidths = 0;
+      num_mbws = 0;
+      num_guardfraction_inputs = 0;
+      int ed_consensus = 0;
+      const uint8_t *ed_consensus_val = NULL;
+
+      /* Okay, go through all the entries for this digest. */
+      for (int voter_idx = 0; voter_idx < smartlist_len(votes); ++voter_idx) {
+        if (vrs_lst[voter_idx] == NULL)
+          continue; /* This voter had nothing to say about this entry. */
+        rs = vrs_lst[voter_idx];
+        ++n_listing;
+
+        current_rsa_id = rs->status.identity_digest;
+
+        smartlist_add(matching_descs, rs);
+        if (rs->version && rs->version[0])
+          smartlist_add(versions, rs->version);
+
+        if (rs->protocols) {
+          /* We include this one even if it's empty: voting for an
+           * empty protocol list actually is meaningful. */
+          smartlist_add(protocols, rs->protocols);
+        }
+
+        /* Tally up all the flags. */
+        for (int flag = 0; flag < n_voter_flags[voter_idx]; ++flag) {
+          if (rs->flags & (UINT64_C(1) << flag))
+            ++flag_counts[flag_map[voter_idx][flag]];
+        }
+        if (named_flag[voter_idx] >= 0 &&
+            (rs->flags & (UINT64_C(1) << named_flag[voter_idx]))) {
+          if (chosen_name && strcmp(chosen_name, rs->status.nickname)) {
+            log_notice(LD_DIR, "Conflict on naming for router: %s vs %s",
+                       chosen_name, rs->status.nickname);
+            naming_conflict = 1;
+          }
+          chosen_name = rs->status.nickname;
+        }
+
+        /* Count guardfraction votes and note down the values. */
+        if (rs->status.has_guardfraction) {
+          measured_guardfraction[num_guardfraction_inputs++] =
+            rs->status.guardfraction_percentage;
+        }
+
+        /* count bandwidths */
+        if (rs->has_measured_bw)
+          measured_bws_kb[num_mbws++] = rs->measured_bw_kb;
+
+        if (rs->status.has_bandwidth)
+          bandwidths_kb[num_bandwidths++] = rs->status.bandwidth_kb;
+
+        /* Count number for which ed25519 is canonical. */
+        if (rs->ed25519_reflects_consensus) {
+          ++ed_consensus;
+          if (ed_consensus_val) {
+            tor_assert(fast_memeq(ed_consensus_val, rs->ed25519_id,
+                                  ED25519_PUBKEY_LEN));
+          } else {
+            ed_consensus_val = rs->ed25519_id;
+          }
+        }
+      }
+
+      /* We don't include this router at all unless more than half of
+       * the authorities we believe in list it. */
+      if (n_listing <= total_authorities/2)
+        continue;
+
+      if (ed_consensus > 0) {
+        if (ed_consensus <= total_authorities / 2) {
+          log_warn(LD_BUG, "Not enough entries had ed_consensus set; how "
+                   "can we have a consensus of %d?", ed_consensus);
+        }
+      }
+
+      /* The clangalyzer can't figure out that this will never be NULL
+       * if n_listing is at least 1 */
+      tor_assert(current_rsa_id);
+
+      /* Figure out the most popular opinion of what the most recent
+       * routerinfo and its contents are. */
+      memset(microdesc_digest, 0, sizeof(microdesc_digest));
+      rs = compute_routerstatus_consensus(matching_descs, consensus_method,
+                                          microdesc_digest, &alt_orport);
+      /* Copy bits of that into rs_out. */
+      memset(&rs_out, 0, sizeof(rs_out));
+      tor_assert(fast_memeq(current_rsa_id,
+                            rs->status.identity_digest,DIGEST_LEN));
+      memcpy(rs_out.identity_digest, current_rsa_id, DIGEST_LEN);
+      memcpy(rs_out.descriptor_digest, rs->status.descriptor_digest,
+             DIGEST_LEN);
+      rs_out.addr = rs->status.addr;
+      rs_out.published_on = rs->status.published_on;
+      rs_out.dir_port = rs->status.dir_port;
+      rs_out.or_port = rs->status.or_port;
+      tor_addr_copy(&rs_out.ipv6_addr, &alt_orport.addr);
+      rs_out.ipv6_orport = alt_orport.port;
+      rs_out.has_bandwidth = 0;
+      rs_out.has_exitsummary = 0;
+
+      if (chosen_name && !naming_conflict) {
+        strlcpy(rs_out.nickname, chosen_name, sizeof(rs_out.nickname));
+      } else {
+        strlcpy(rs_out.nickname, rs->status.nickname, sizeof(rs_out.nickname));
+      }
+
+      {
+        const char *d = strmap_get_lc(name_to_id_map, rs_out.nickname);
+        if (!d) {
+          is_named = is_unnamed = 0;
+        } else if (fast_memeq(d, current_rsa_id, DIGEST_LEN)) {
+          is_named = 1; is_unnamed = 0;
+        } else {
+          is_named = 0; is_unnamed = 1;
+        }
+      }
+
+      /* Set the flags. */
+      smartlist_add(chosen_flags, (char*)"s"); /* for the start of the line. */
+      SMARTLIST_FOREACH_BEGIN(flags, const char *, fl) {
+        if (!strcmp(fl, "Named")) {
+          if (is_named)
+            smartlist_add(chosen_flags, (char*)fl);
+        } else if (!strcmp(fl, "Unnamed")) {
+          if (is_unnamed)
+            smartlist_add(chosen_flags, (char*)fl);
+        } else if (!strcmp(fl, "NoEdConsensus")) {
+          if (ed_consensus <= total_authorities/2)
+            smartlist_add(chosen_flags, (char*)fl);
+        } else {
+          if (flag_counts[fl_sl_idx] > n_flag_voters[fl_sl_idx]/2) {
+            smartlist_add(chosen_flags, (char*)fl);
+            if (!strcmp(fl, "Exit"))
+              is_exit = 1;
+            else if (!strcmp(fl, "Guard"))
+              is_guard = 1;
+            else if (!strcmp(fl, "Running"))
+              is_running = 1;
+            else if (!strcmp(fl, "BadExit"))
+              is_bad_exit = 1;
+            else if (!strcmp(fl, "Valid"))
+              is_valid = 1;
+          }
+        }
+      } SMARTLIST_FOREACH_END(fl);
+
+      /* Starting with consensus method 4 we do not list servers
+       * that are not running in a consensus.  See Proposal 138 */
+      if (!is_running)
+        continue;
+
+      /* Starting with consensus method 24, we don't list servers
+       * that are not valid in a consensus.  See Proposal 272 */
+      if (!is_valid)
+        continue;
+
+      /* Pick the version. */
+      if (smartlist_len(versions)) {
+        sort_version_list(versions, 0);
+        chosen_version = get_most_frequent_member(versions);
+      } else {
+        chosen_version = NULL;
+      }
+
+      /* Pick the protocol list */
+      if (smartlist_len(protocols)) {
+        smartlist_sort_strings(protocols);
+        chosen_protocol_list = get_most_frequent_member(protocols);
+      } else {
+        chosen_protocol_list = NULL;
+      }
+
+      /* If it's a guard and we have enough guardfraction votes,
+         calculate its consensus guardfraction value. */
+      if (is_guard && num_guardfraction_inputs > 2) {
+        rs_out.has_guardfraction = 1;
+        rs_out.guardfraction_percentage = median_uint32(measured_guardfraction,
+                                                     num_guardfraction_inputs);
+        /* final value should be an integer percentage! */
+        tor_assert(rs_out.guardfraction_percentage <= 100);
+      }
+
+      /* Pick a bandwidth */
+      if (num_mbws > 2) {
+        rs_out.has_bandwidth = 1;
+        rs_out.bw_is_unmeasured = 0;
+        rs_out.bandwidth_kb = median_uint32(measured_bws_kb, num_mbws);
+      } else if (num_bandwidths > 0) {
+        rs_out.has_bandwidth = 1;
+        rs_out.bw_is_unmeasured = 1;
+        rs_out.bandwidth_kb = median_uint32(bandwidths_kb, num_bandwidths);
+        if (n_authorities_measuring_bandwidth > 2) {
+          /* Cap non-measured bandwidths. */
+          if (rs_out.bandwidth_kb > max_unmeasured_bw_kb) {
+            rs_out.bandwidth_kb = max_unmeasured_bw_kb;
+          }
+        }
+      }
+
+      /* Fix bug 2203: Do not count BadExit nodes as Exits for bw weights */
+      is_exit = is_exit && !is_bad_exit;
+
+      /* Update total bandwidth weights with the bandwidths of this router. */
+      {
+        update_total_bandwidth_weights(&rs_out,
+                                       is_exit, is_guard,
+                                       &G, &M, &E, &D, &T);
+      }
+
+      /* Ok, we already picked a descriptor digest we want to list
+       * previously.  Now we want to use the exit policy summary from
+       * that descriptor.  If everybody plays nice all the voters who
+       * listed that descriptor will have the same summary.  If not then
+       * something is fishy and we'll use the most common one (breaking
+       * ties in favor of lexicographically larger one (only because it
+       * lets me reuse more existing code)).
+       *
+       * The other case that can happen is that no authority that voted
+       * for that descriptor has an exit policy summary.  That's
+       * probably quite unlikely but can happen.  In that case we use
+       * the policy that was most often listed in votes, again breaking
+       * ties like in the previous case.
+       */
+      {
+        /* Okay, go through all the votes for this router.  We prepared
+         * that list previously */
+        const char *chosen_exitsummary = NULL;
+        smartlist_clear(exitsummaries);
+        SMARTLIST_FOREACH_BEGIN(matching_descs, vote_routerstatus_t *, vsr) {
+          /* Check if the vote where this status comes from had the
+           * proper descriptor */
+          tor_assert(fast_memeq(rs_out.identity_digest,
+                             vsr->status.identity_digest,
+                             DIGEST_LEN));
+          if (vsr->status.has_exitsummary &&
+               fast_memeq(rs_out.descriptor_digest,
+                       vsr->status.descriptor_digest,
+                       DIGEST_LEN)) {
+            tor_assert(vsr->status.exitsummary);
+            smartlist_add(exitsummaries, vsr->status.exitsummary);
+            if (!chosen_exitsummary) {
+              chosen_exitsummary = vsr->status.exitsummary;
+            } else if (strcmp(chosen_exitsummary, vsr->status.exitsummary)) {
+              /* Great.  There's disagreement among the voters.  That
+               * really shouldn't be */
+              exitsummary_disagreement = 1;
+            }
+          }
+        } SMARTLIST_FOREACH_END(vsr);
+
+        if (exitsummary_disagreement) {
+          char id[HEX_DIGEST_LEN+1];
+          char dd[HEX_DIGEST_LEN+1];
+          base16_encode(id, sizeof(dd), rs_out.identity_digest, DIGEST_LEN);
+          base16_encode(dd, sizeof(dd), rs_out.descriptor_digest, DIGEST_LEN);
+          log_warn(LD_DIR, "The voters disagreed on the exit policy summary "
+                   " for router %s with descriptor %s.  This really shouldn't"
+                   " have happened.", id, dd);
+
+          smartlist_sort_strings(exitsummaries);
+          chosen_exitsummary = get_most_frequent_member(exitsummaries);
+        } else if (!chosen_exitsummary) {
+          char id[HEX_DIGEST_LEN+1];
+          char dd[HEX_DIGEST_LEN+1];
+          base16_encode(id, sizeof(dd), rs_out.identity_digest, DIGEST_LEN);
+          base16_encode(dd, sizeof(dd), rs_out.descriptor_digest, DIGEST_LEN);
+          log_warn(LD_DIR, "Not one of the voters that made us select"
+                   "descriptor %s for router %s had an exit policy"
+                   "summary", dd, id);
+
+          /* Ok, none of those voting for the digest we chose had an
+           * exit policy for us.  Well, that kinda sucks.
+           */
+          smartlist_clear(exitsummaries);
+          SMARTLIST_FOREACH(matching_descs, vote_routerstatus_t *, vsr, {
+            if (vsr->status.has_exitsummary)
+              smartlist_add(exitsummaries, vsr->status.exitsummary);
+          });
+          smartlist_sort_strings(exitsummaries);
+          chosen_exitsummary = get_most_frequent_member(exitsummaries);
+
+          if (!chosen_exitsummary)
+            log_warn(LD_DIR, "Wow, not one of the voters had an exit "
+                     "policy summary for %s.  Wow.", id);
+        }
+
+        if (chosen_exitsummary) {
+          rs_out.has_exitsummary = 1;
+          /* yea, discards the const */
+          rs_out.exitsummary = (char *)chosen_exitsummary;
+        }
+      }
+
+      if (flavor == FLAV_MICRODESC &&
+          tor_digest256_is_zero(microdesc_digest)) {
+        /* With no microdescriptor digest, we omit the entry entirely. */
+        continue;
+      }
+
+      {
+        char *buf;
+        /* Okay!! Now we can write the descriptor... */
+        /*     First line goes into "buf". */
+        buf = routerstatus_format_entry(&rs_out, NULL, NULL,
+                                        rs_format, consensus_method, NULL);
+        if (buf)
+          smartlist_add(chunks, buf);
+      }
+      /*     Now an m line, if applicable. */
+      if (flavor == FLAV_MICRODESC &&
+          !tor_digest256_is_zero(microdesc_digest)) {
+        char m[BASE64_DIGEST256_LEN+1];
+        digest256_to_base64(m, microdesc_digest);
+        smartlist_add_asprintf(chunks, "m %s\n", m);
+      }
+      /*     Next line is all flags.  The "\n" is missing. */
+      smartlist_add(chunks,
+                    smartlist_join_strings(chosen_flags, " ", 0, NULL));
+      /*     Now the version line. */
+      if (chosen_version) {
+        smartlist_add_strdup(chunks, "\nv ");
+        smartlist_add_strdup(chunks, chosen_version);
+      }
+      smartlist_add_strdup(chunks, "\n");
+      if (chosen_protocol_list &&
+          consensus_method >= MIN_METHOD_FOR_RS_PROTOCOLS) {
+        smartlist_add_asprintf(chunks, "pr %s\n", chosen_protocol_list);
+      }
+      /*     Now the weight line. */
+      if (rs_out.has_bandwidth) {
+        char *guardfraction_str = NULL;
+        int unmeasured = rs_out.bw_is_unmeasured;
+
+        /* If we have guardfraction info, include it in the 'w' line. */
+        if (rs_out.has_guardfraction) {
+          tor_asprintf(&guardfraction_str,
+                       " GuardFraction=%u", rs_out.guardfraction_percentage);
+        }
+        smartlist_add_asprintf(chunks, "w Bandwidth=%d%s%s\n",
+                               rs_out.bandwidth_kb,
+                               unmeasured?" Unmeasured=1":"",
+                               guardfraction_str ? guardfraction_str : "");
+
+        tor_free(guardfraction_str);
+      }
+
+      /*     Now the exitpolicy summary line. */
+      if (rs_out.has_exitsummary && flavor == FLAV_NS) {
+        smartlist_add_asprintf(chunks, "p %s\n", rs_out.exitsummary);
+      }
+
+      /* And the loop is over and we move on to the next router */
+    }
+
+    tor_free(size);
+    tor_free(n_voter_flags);
+    tor_free(n_flag_voters);
+    for (i = 0; i < smartlist_len(votes); ++i)
+      tor_free(flag_map[i]);
+    tor_free(flag_map);
+    tor_free(flag_counts);
+    tor_free(named_flag);
+    tor_free(unnamed_flag);
+    strmap_free(name_to_id_map, NULL);
+    smartlist_free(matching_descs);
+    smartlist_free(chosen_flags);
+    smartlist_free(versions);
+    smartlist_free(protocols);
+    smartlist_free(exitsummaries);
+    tor_free(bandwidths_kb);
+    tor_free(measured_bws_kb);
+    tor_free(measured_guardfraction);
+  }
+
+  /* Mark the directory footer region */
+  smartlist_add_strdup(chunks, "directory-footer\n");
+
+  {
+    int64_t weight_scale = BW_WEIGHT_SCALE;
+    char *bw_weight_param = NULL;
+
+    // Parse params, extract BW_WEIGHT_SCALE if present
+    // DO NOT use consensus_param_bw_weight_scale() in this code!
+    // The consensus is not formed yet!
+    /* XXXX Extract this code into a common function. Or not: #19011. */
+    if (params) {
+      if (strcmpstart(params, "bwweightscale=") == 0)
+        bw_weight_param = params;
+      else
+        bw_weight_param = strstr(params, " bwweightscale=");
+    }
+
+    if (bw_weight_param) {
+      int ok=0;
+      char *eq = strchr(bw_weight_param, '=');
+      if (eq) {
+        weight_scale = tor_parse_long(eq+1, 10, 1, INT32_MAX, &ok,
+                                         NULL);
+        if (!ok) {
+          log_warn(LD_DIR, "Bad element '%s' in bw weight param",
+              escaped(bw_weight_param));
+          weight_scale = BW_WEIGHT_SCALE;
+        }
+      } else {
+        log_warn(LD_DIR, "Bad element '%s' in bw weight param",
+            escaped(bw_weight_param));
+        weight_scale = BW_WEIGHT_SCALE;
+      }
+    }
+
+    added_weights = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D,
+                                                         T, weight_scale);
+  }
+
+  /* Add a signature. */
+  {
+    char digest[DIGEST256_LEN];
+    char fingerprint[HEX_DIGEST_LEN+1];
+    char signing_key_fingerprint[HEX_DIGEST_LEN+1];
+    digest_algorithm_t digest_alg =
+      flavor == FLAV_NS ? DIGEST_SHA1 : DIGEST_SHA256;
+    size_t digest_len =
+      flavor == FLAV_NS ? DIGEST_LEN : DIGEST256_LEN;
+    const char *algname = crypto_digest_algorithm_get_name(digest_alg);
+    char *signature;
+
+    smartlist_add_strdup(chunks, "directory-signature ");
+
+    /* Compute the hash of the chunks. */
+    crypto_digest_smartlist(digest, digest_len, chunks, "", digest_alg);
+
+    /* Get the fingerprints */
+    crypto_pk_get_fingerprint(identity_key, fingerprint, 0);
+    crypto_pk_get_fingerprint(signing_key, signing_key_fingerprint, 0);
+
+    /* add the junk that will go at the end of the line. */
+    if (flavor == FLAV_NS) {
+      smartlist_add_asprintf(chunks, "%s %s\n", fingerprint,
+                   signing_key_fingerprint);
+    } else {
+      smartlist_add_asprintf(chunks, "%s %s %s\n",
+                   algname, fingerprint,
+                   signing_key_fingerprint);
+    }
+    /* And the signature. */
+    if (!(signature = router_get_dirobj_signature(digest, digest_len,
+                                                  signing_key))) {
+      log_warn(LD_BUG, "Couldn't sign consensus networkstatus.");
+      goto done;
+    }
+    smartlist_add(chunks, signature);
+
+    if (legacy_id_key_digest && legacy_signing_key) {
+      smartlist_add_strdup(chunks, "directory-signature ");
+      base16_encode(fingerprint, sizeof(fingerprint),
+                    legacy_id_key_digest, DIGEST_LEN);
+      crypto_pk_get_fingerprint(legacy_signing_key,
+                                signing_key_fingerprint, 0);
+      if (flavor == FLAV_NS) {
+        smartlist_add_asprintf(chunks, "%s %s\n", fingerprint,
+                     signing_key_fingerprint);
+      } else {
+        smartlist_add_asprintf(chunks, "%s %s %s\n",
+                     algname, fingerprint,
+                     signing_key_fingerprint);
+      }
+
+      if (!(signature = router_get_dirobj_signature(digest, digest_len,
+                                                    legacy_signing_key))) {
+        log_warn(LD_BUG, "Couldn't sign consensus networkstatus.");
+        goto done;
+      }
+      smartlist_add(chunks, signature);
+    }
+  }
+
+  result = smartlist_join_strings(chunks, "", 0, NULL);
+
+  {
+    networkstatus_t *c;
+    if (!(c = networkstatus_parse_vote_from_string(result, NULL,
+                                                   NS_TYPE_CONSENSUS))) {
+      log_err(LD_BUG, "Generated a networkstatus consensus we couldn't "
+              "parse.");
+      tor_free(result);
+      goto done;
+    }
+    // Verify balancing parameters
+    if (added_weights) {
+      networkstatus_verify_bw_weights(c, consensus_method);
+    }
+    networkstatus_vote_free(c);
+  }
+
+ done:
+
+  dircollator_free(collator);
+  tor_free(client_versions);
+  tor_free(server_versions);
+  tor_free(packages);
+  SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
+  smartlist_free(flags);
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+  SMARTLIST_FOREACH(param_list, char *, cp, tor_free(cp));
+  smartlist_free(param_list);
+
+  return result;
+}
+
+/** Given a list of networkstatus_t for each vote, return a newly allocated
+ * string containing the "package" lines for the vote. */
+STATIC char *
+compute_consensus_package_lines(smartlist_t *votes)
+{
+  const int n_votes = smartlist_len(votes);
+
+  /* This will be a map from "packagename version" strings to arrays
+   * of const char *, with the i'th member of the array corresponding to the
+   * package line from the i'th vote.
+   */
+  strmap_t *package_status = strmap_new();
+
+  SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+    if (! v->package_lines)
+      continue;
+    SMARTLIST_FOREACH_BEGIN(v->package_lines, const char *, line) {
+      if (! validate_recommended_package_line(line))
+        continue;
+
+      /* Skip 'cp' to the second space in the line. */
+      const char *cp = strchr(line, ' ');
+      if (!cp) continue;
+      ++cp;
+      cp = strchr(cp, ' ');
+      if (!cp) continue;
+
+      char *key = tor_strndup(line, cp - line);
+
+      const char **status = strmap_get(package_status, key);
+      if (!status) {
+        status = tor_calloc(n_votes, sizeof(const char *));
+        strmap_set(package_status, key, status);
+      }
+      status[v_sl_idx] = line; /* overwrite old value */
+      tor_free(key);
+    } SMARTLIST_FOREACH_END(line);
+  } SMARTLIST_FOREACH_END(v);
+
+  smartlist_t *entries = smartlist_new(); /* temporary */
+  smartlist_t *result_list = smartlist_new(); /* output */
+  STRMAP_FOREACH(package_status, key, const char **, values) {
+    int i, count=-1;
+    for (i = 0; i < n_votes; ++i) {
+      if (values[i])
+        smartlist_add(entries, (void*) values[i]);
+    }
+    smartlist_sort_strings(entries);
+    int n_voting_for_entry = smartlist_len(entries);
+    const char *most_frequent =
+      smartlist_get_most_frequent_string_(entries, &count);
+
+    if (n_voting_for_entry >= 3 && count > n_voting_for_entry / 2) {
+      smartlist_add_asprintf(result_list, "package %s\n", most_frequent);
+    }
+
+    smartlist_clear(entries);
+
+  } STRMAP_FOREACH_END;
+
+  smartlist_sort_strings(result_list);
+
+  char *result = smartlist_join_strings(result_list, "", 0, NULL);
+
+  SMARTLIST_FOREACH(result_list, char *, cp, tor_free(cp));
+  smartlist_free(result_list);
+  smartlist_free(entries);
+  strmap_free(package_status, tor_free_);
+
+  return result;
+}
+
+/** Given a consensus vote <b>target</b> and a set of detached signatures in
+ * <b>sigs</b> that correspond to the same consensus, check whether there are
+ * any new signatures in <b>src_voter_list</b> that should be added to
+ * <b>target</b>. (A signature should be added if we have no signature for that
+ * voter in <b>target</b> yet, or if we have no verifiable signature and the
+ * new signature is verifiable.)  Return the number of signatures added or
+ * changed, or -1 if the document signed by <b>sigs</b> isn't the same
+ * document as <b>target</b>. */
+STATIC int
+networkstatus_add_detached_signatures(networkstatus_t *target,
+                                      ns_detached_signatures_t *sigs,
+                                      const char *source,
+                                      int severity,
+                                      const char **msg_out)
+{
+  int r = 0;
+  const char *flavor;
+  smartlist_t *siglist;
+  tor_assert(sigs);
+  tor_assert(target);
+  tor_assert(target->type == NS_TYPE_CONSENSUS);
+
+  flavor = networkstatus_get_flavor_name(target->flavor);
+
+  /* Do the times seem right? */
+  if (target->valid_after != sigs->valid_after) {
+    *msg_out = "Valid-After times do not match "
+      "when adding detached signatures to consensus";
+    return -1;
+  }
+  if (target->fresh_until != sigs->fresh_until) {
+    *msg_out = "Fresh-until times do not match "
+      "when adding detached signatures to consensus";
+    return -1;
+  }
+  if (target->valid_until != sigs->valid_until) {
+    *msg_out = "Valid-until times do not match "
+      "when adding detached signatures to consensus";
+    return -1;
+  }
+  siglist = strmap_get(sigs->signatures, flavor);
+  if (!siglist) {
+    *msg_out = "No signatures for given consensus flavor";
+    return -1;
+  }
+
+  /** Make sure all the digests we know match, and at least one matches. */
+  {
+    common_digests_t *digests = strmap_get(sigs->digests, flavor);
+    int n_matches = 0;
+    int alg;
+    if (!digests) {
+      *msg_out = "No digests for given consensus flavor";
+      return -1;
+    }
+    for (alg = DIGEST_SHA1; alg < N_COMMON_DIGEST_ALGORITHMS; ++alg) {
+      if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) {
+        if (fast_memeq(target->digests.d[alg], digests->d[alg],
+                       DIGEST256_LEN)) {
+          ++n_matches;
+        } else {
+          *msg_out = "Mismatched digest.";
+          return -1;
+        }
+      }
+    }
+    if (!n_matches) {
+      *msg_out = "No recognized digests for given consensus flavor";
+    }
+  }
+
+  /* For each voter in src... */
+  SMARTLIST_FOREACH_BEGIN(siglist, document_signature_t *, sig) {
+    char voter_identity[HEX_DIGEST_LEN+1];
+    networkstatus_voter_info_t *target_voter =
+      networkstatus_get_voter_by_id(target, sig->identity_digest);
+    authority_cert_t *cert = NULL;
+    const char *algorithm;
+    document_signature_t *old_sig = NULL;
+
+    algorithm = crypto_digest_algorithm_get_name(sig->alg);
+
+    base16_encode(voter_identity, sizeof(voter_identity),
+                  sig->identity_digest, DIGEST_LEN);
+    log_info(LD_DIR, "Looking at signature from %s using %s", voter_identity,
+             algorithm);
+    /* If the target doesn't know about this voter, then forget it. */
+    if (!target_voter) {
+      log_info(LD_DIR, "We do not know any voter with ID %s", voter_identity);
+      continue;
+    }
+
+    old_sig = networkstatus_get_voter_sig_by_alg(target_voter, sig->alg);
+
+    /* If the target already has a good signature from this voter, then skip
+     * this one. */
+    if (old_sig && old_sig->good_signature) {
+      log_info(LD_DIR, "We already have a good signature from %s using %s",
+               voter_identity, algorithm);
+      continue;
+    }
+
+    /* Try checking the signature if we haven't already. */
+    if (!sig->good_signature && !sig->bad_signature) {
+      cert = authority_cert_get_by_digests(sig->identity_digest,
+                                           sig->signing_key_digest);
+      if (cert) {
+        /* Not checking the return value here, since we are going to look
+         * at the status of sig->good_signature in a moment. */
+        (void) networkstatus_check_document_signature(target, sig, cert);
+      }
+    }
+
+    /* If this signature is good, or we don't have any signature yet,
+     * then maybe add it. */
+    if (sig->good_signature || !old_sig || old_sig->bad_signature) {
+      log_info(LD_DIR, "Adding signature from %s with %s", voter_identity,
+               algorithm);
+      tor_log(severity, LD_DIR, "Added a signature for %s from %s.",
+          target_voter->nickname, source);
+      ++r;
+      if (old_sig) {
+        smartlist_remove(target_voter->sigs, old_sig);
+        document_signature_free(old_sig);
+      }
+      smartlist_add(target_voter->sigs, document_signature_dup(sig));
+    } else {
+      log_info(LD_DIR, "Not adding signature from %s", voter_identity);
+    }
+  } SMARTLIST_FOREACH_END(sig);
+
+  return r;
+}
+
+/** Return a newly allocated string containing all the signatures on
+ * <b>consensus</b> by all voters. If <b>for_detached_signatures</b> is true,
+ * then the signatures will be put in a detached signatures document, so
+ * prefix any non-NS-flavored signatures with "additional-signature" rather
+ * than "directory-signature". */
+static char *
+networkstatus_format_signatures(networkstatus_t *consensus,
+                                int for_detached_signatures)
+{
+  smartlist_t *elements;
+  char buf[4096];
+  char *result = NULL;
+  int n_sigs = 0;
+  const consensus_flavor_t flavor = consensus->flavor;
+  const char *flavor_name = networkstatus_get_flavor_name(flavor);
+  const char *keyword;
+
+  if (for_detached_signatures && flavor != FLAV_NS)
+    keyword = "additional-signature";
+  else
+    keyword = "directory-signature";
+
+  elements = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) {
+    SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) {
+      char sk[HEX_DIGEST_LEN+1];
+      char id[HEX_DIGEST_LEN+1];
+      if (!sig->signature || sig->bad_signature)
+        continue;
+      ++n_sigs;
+      base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN);
+      base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN);
+      if (flavor == FLAV_NS) {
+        smartlist_add_asprintf(elements,
+                     "%s %s %s\n-----BEGIN SIGNATURE-----\n",
+                     keyword, id, sk);
+      } else {
+        const char *digest_name =
+          crypto_digest_algorithm_get_name(sig->alg);
+        smartlist_add_asprintf(elements,
+                     "%s%s%s %s %s %s\n-----BEGIN SIGNATURE-----\n",
+                     keyword,
+                     for_detached_signatures ? " " : "",
+                     for_detached_signatures ? flavor_name : "",
+                     digest_name, id, sk);
+      }
+      base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len,
+                    BASE64_ENCODE_MULTILINE);
+      strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf));
+      smartlist_add_strdup(elements, buf);
+    } SMARTLIST_FOREACH_END(sig);
+  } SMARTLIST_FOREACH_END(v);
+
+  result = smartlist_join_strings(elements, "", 0, NULL);
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+  if (!n_sigs)
+    tor_free(result);
+  return result;
+}
+
+/** Return a newly allocated string holding the detached-signatures document
+ * corresponding to the signatures on <b>consensuses</b>, which must contain
+ * exactly one FLAV_NS consensus, and no more than one consensus for each
+ * other flavor. */
+STATIC char *
+networkstatus_get_detached_signatures(smartlist_t *consensuses)
+{
+  smartlist_t *elements;
+  char *result = NULL, *sigs = NULL;
+  networkstatus_t *consensus_ns = NULL;
+  tor_assert(consensuses);
+
+  SMARTLIST_FOREACH(consensuses, networkstatus_t *, ns, {
+      tor_assert(ns);
+      tor_assert(ns->type == NS_TYPE_CONSENSUS);
+      if (ns && ns->flavor == FLAV_NS)
+        consensus_ns = ns;
+  });
+  if (!consensus_ns) {
+    log_warn(LD_BUG, "No NS consensus given.");
+    return NULL;
+  }
+
+  elements = smartlist_new();
+
+  {
+    char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1],
+      vu_buf[ISO_TIME_LEN+1];
+    char d[HEX_DIGEST_LEN+1];
+
+    base16_encode(d, sizeof(d),
+                  consensus_ns->digests.d[DIGEST_SHA1], DIGEST_LEN);
+    format_iso_time(va_buf, consensus_ns->valid_after);
+    format_iso_time(fu_buf, consensus_ns->fresh_until);
+    format_iso_time(vu_buf, consensus_ns->valid_until);
+
+    smartlist_add_asprintf(elements,
+                 "consensus-digest %s\n"
+                 "valid-after %s\n"
+                 "fresh-until %s\n"
+                 "valid-until %s\n", d, va_buf, fu_buf, vu_buf);
+  }
+
+  /* Get all the digests for the non-FLAV_NS consensuses */
+  SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) {
+    const char *flavor_name = networkstatus_get_flavor_name(ns->flavor);
+    int alg;
+    if (ns->flavor == FLAV_NS)
+      continue;
+
+    /* start with SHA256; we don't include SHA1 for anything but the basic
+     * consensus. */
+    for (alg = DIGEST_SHA256; alg < N_COMMON_DIGEST_ALGORITHMS; ++alg) {
+      char d[HEX_DIGEST256_LEN+1];
+      const char *alg_name =
+        crypto_digest_algorithm_get_name(alg);
+      if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN))
+        continue;
+      base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN);
+      smartlist_add_asprintf(elements, "additional-digest %s %s %s\n",
+                   flavor_name, alg_name, d);
+    }
+  } SMARTLIST_FOREACH_END(ns);
+
+  /* Now get all the sigs for non-FLAV_NS consensuses */
+  SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) {
+    char *sigs_on_this_consensus;
+    if (ns->flavor == FLAV_NS)
+      continue;
+    sigs_on_this_consensus = networkstatus_format_signatures(ns, 1);
+    if (!sigs_on_this_consensus) {
+      log_warn(LD_DIR, "Couldn't format signatures");
+      goto err;
+    }
+    smartlist_add(elements, sigs_on_this_consensus);
+  } SMARTLIST_FOREACH_END(ns);
+
+  /* Now add the FLAV_NS consensus signatrures. */
+  sigs = networkstatus_format_signatures(consensus_ns, 1);
+  if (!sigs)
+    goto err;
+  smartlist_add(elements, sigs);
+
+  result = smartlist_join_strings(elements, "", 0, NULL);
+ err:
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+  return result;
+}
+
+/** Return a newly allocated string holding a detached-signatures document for
+ * all of the in-progress consensuses in the <b>n_flavors</b>-element array at
+ * <b>pending</b>. */
+static char *
+get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending,
+                                                 int n_flavors)
+{
+  int flav;
+  char *signatures;
+  smartlist_t *c = smartlist_new();
+  for (flav = 0; flav < n_flavors; ++flav) {
+    if (pending[flav].consensus)
+      smartlist_add(c, pending[flav].consensus);
+  }
+  signatures = networkstatus_get_detached_signatures(c);
+  smartlist_free(c);
+  return signatures;
+}
+
+/**
+ * Entry point: Take whatever voting actions are pending as of <b>now</b>.
+ *
+ * Return the time at which the next action should be taken.
+ */
+time_t
+dirvote_act(const or_options_t *options, time_t now)
+{
+  if (!authdir_mode_v3(options))
+    return TIME_MAX;
+  tor_assert_nonfatal(voting_schedule.voting_starts);
+  /* If we haven't initialized this object through this codeflow, we need to
+   * recalculate the timings to match our vote. The reason to do that is if we
+   * have a voting schedule initialized 1 minute ago, the voting timings might
+   * not be aligned to what we should expect with "now". This is especially
+   * true for TestingTorNetwork using smaller timings.  */
+  if (voting_schedule.created_on_demand) {
+    char *keys = list_v3_auth_ids();
+    authority_cert_t *c = get_my_v3_authority_cert();
+    log_notice(LD_DIR, "Scheduling voting.  Known authority IDs are %s. "
+               "Mine is %s.",
+               keys, hex_str(c->cache_info.identity_digest, DIGEST_LEN));
+    tor_free(keys);
+    voting_schedule_recalculate_timing(options, now);
+  }
+
+#define IF_TIME_FOR_NEXT_ACTION(when_field, done_field) \
+  if (! voting_schedule.done_field) {                   \
+    if (voting_schedule.when_field > now) {             \
+      return voting_schedule.when_field;                \
+    } else {
+#define ENDIF \
+    }           \
+  }
+
+  IF_TIME_FOR_NEXT_ACTION(voting_starts, have_voted) {
+    log_notice(LD_DIR, "Time to vote.");
+    dirvote_perform_vote();
+    voting_schedule.have_voted = 1;
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(fetch_missing_votes, have_fetched_missing_votes) {
+    log_notice(LD_DIR, "Time to fetch any votes that we're missing.");
+    dirvote_fetch_missing_votes();
+    voting_schedule.have_fetched_missing_votes = 1;
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(voting_ends, have_built_consensus) {
+    log_notice(LD_DIR, "Time to compute a consensus.");
+    dirvote_compute_consensuses();
+    /* XXXX We will want to try again later if we haven't got enough
+     * votes yet.  Implement this if it turns out to ever happen. */
+    voting_schedule.have_built_consensus = 1;
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(fetch_missing_signatures,
+                          have_fetched_missing_signatures) {
+    log_notice(LD_DIR, "Time to fetch any signatures that we're missing.");
+    dirvote_fetch_missing_signatures();
+    voting_schedule.have_fetched_missing_signatures = 1;
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(interval_starts,
+                          have_published_consensus) {
+    log_notice(LD_DIR, "Time to publish the consensus and discard old votes");
+    dirvote_publish_consensus();
+    dirvote_clear_votes(0);
+    voting_schedule.have_published_consensus = 1;
+    /* Update our shared random state with the consensus just published. */
+    sr_act_post_consensus(
+                networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
+    /* XXXX We will want to try again later if we haven't got enough
+     * signatures yet.  Implement this if it turns out to ever happen. */
+    voting_schedule_recalculate_timing(options, now);
+    return voting_schedule.voting_starts;
+  } ENDIF
+
+  tor_assert_nonfatal_unreached();
+  return now + 1;
+
+#undef ENDIF
+#undef IF_TIME_FOR_NEXT_ACTION
+}
+
+/** A vote networkstatus_t and its unparsed body: held around so we can
+ * use it to generate a consensus (at voting_ends) and so we can serve it to
+ * other authorities that might want it. */
+typedef struct pending_vote_t {
+  cached_dir_t *vote_body;
+  networkstatus_t *vote;
+} pending_vote_t;
+
+/** List of pending_vote_t for the current vote.  Before we've used them to
+ * build a consensus, the votes go here. */
+static smartlist_t *pending_vote_list = NULL;
+/** List of pending_vote_t for the previous vote.  After we've used them to
+ * build a consensus, the votes go here for the next period. */
+static smartlist_t *previous_vote_list = NULL;
+
+/* DOCDOC pending_consensuses */
+static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS];
+
+/** The detached signatures for the consensus that we're currently
+ * building. */
+static char *pending_consensus_signatures = NULL;
+
+/** List of ns_detached_signatures_t: hold signatures that get posted to us
+ * before we have generated the consensus on our own. */
+static smartlist_t *pending_consensus_signature_list = NULL;
+
+/** Generate a networkstatus vote and post it to all the v3 authorities.
+ * (V3 Authority only) */
+static int
+dirvote_perform_vote(void)
+{
+  crypto_pk_t *key = get_my_v3_authority_signing_key();
+  authority_cert_t *cert = get_my_v3_authority_cert();
+  networkstatus_t *ns;
+  char *contents;
+  pending_vote_t *pending_vote;
+  time_t now = time(NULL);
+
+  int status;
+  const char *msg = "";
+
+  if (!cert || !key) {
+    log_warn(LD_NET, "Didn't find key/certificate to generate v3 vote");
+    return -1;
+  } else if (cert->expires < now) {
+    log_warn(LD_NET, "Can't generate v3 vote with expired certificate");
+    return -1;
+  }
+  if (!(ns = dirserv_generate_networkstatus_vote_obj(key, cert)))
+    return -1;
+
+  contents = format_networkstatus_vote(key, ns);
+  networkstatus_vote_free(ns);
+  if (!contents)
+    return -1;
+
+  pending_vote = dirvote_add_vote(contents, &msg, &status);
+  tor_free(contents);
+  if (!pending_vote) {
+    log_warn(LD_DIR, "Couldn't store my own vote! (I told myself, '%s'.)",
+             msg);
+    return -1;
+  }
+
+  directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_VOTE,
+                               ROUTER_PURPOSE_GENERAL,
+                               V3_DIRINFO,
+                               pending_vote->vote_body->dir,
+                               pending_vote->vote_body->dir_len, 0);
+  log_notice(LD_DIR, "Vote posted.");
+  return 0;
+}
+
+/** Send an HTTP request to every other v3 authority, for the votes of every
+ * authority for which we haven't received a vote yet in this period. (V3
+ * authority only) */
+static void
+dirvote_fetch_missing_votes(void)
+{
+  smartlist_t *missing_fps = smartlist_new();
+  char *resource;
+
+  SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
+                          dir_server_t *, ds) {
+      if (!(ds->type & V3_DIRINFO))
+        continue;
+      if (!dirvote_get_vote(ds->v3_identity_digest,
+                            DGV_BY_ID|DGV_INCLUDE_PENDING)) {
+        char *cp = tor_malloc(HEX_DIGEST_LEN+1);
+        base16_encode(cp, HEX_DIGEST_LEN+1, ds->v3_identity_digest,
+                      DIGEST_LEN);
+        smartlist_add(missing_fps, cp);
+      }
+  } SMARTLIST_FOREACH_END(ds);
+
+  if (!smartlist_len(missing_fps)) {
+    smartlist_free(missing_fps);
+    return;
+  }
+  {
+    char *tmp = smartlist_join_strings(missing_fps, " ", 0, NULL);
+    log_notice(LOG_NOTICE, "We're missing votes from %d authorities (%s). "
+               "Asking every other authority for a copy.",
+               smartlist_len(missing_fps), tmp);
+    tor_free(tmp);
+  }
+  resource = smartlist_join_strings(missing_fps, "+", 0, NULL);
+  directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE,
+                                     0, resource);
+  tor_free(resource);
+  SMARTLIST_FOREACH(missing_fps, char *, cp, tor_free(cp));
+  smartlist_free(missing_fps);
+}
+
+/** Send a request to every other authority for its detached signatures,
+ * unless we have signatures from all other v3 authorities already. */
+static void
+dirvote_fetch_missing_signatures(void)
+{
+  int need_any = 0;
+  int i;
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
+    networkstatus_t *consensus = pending_consensuses[i].consensus;
+    if (!consensus ||
+        networkstatus_check_consensus_signature(consensus, -1) == 1) {
+      /* We have no consensus, or we have one that's signed by everybody. */
+      continue;
+    }
+    need_any = 1;
+  }
+  if (!need_any)
+    return;
+
+  directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES,
+                                     0, NULL);
+}
+
+/** Release all storage held by pending consensuses (those waiting for
+ * signatures). */
+static void
+dirvote_clear_pending_consensuses(void)
+{
+  int i;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    pending_consensus_t *pc = &pending_consensuses[i];
+    tor_free(pc->body);
+
+    networkstatus_vote_free(pc->consensus);
+    pc->consensus = NULL;
+  }
+}
+
+/** Drop all currently pending votes, consensus, and detached signatures. */
+static void
+dirvote_clear_votes(int all_votes)
+{
+  if (!previous_vote_list)
+    previous_vote_list = smartlist_new();
+  if (!pending_vote_list)
+    pending_vote_list = smartlist_new();
+
+  /* All "previous" votes are now junk. */
+  SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, v, {
+      cached_dir_decref(v->vote_body);
+      v->vote_body = NULL;
+      networkstatus_vote_free(v->vote);
+      tor_free(v);
+    });
+  smartlist_clear(previous_vote_list);
+
+  if (all_votes) {
+    /* If we're dumping all the votes, we delete the pending ones. */
+    SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, {
+        cached_dir_decref(v->vote_body);
+        v->vote_body = NULL;
+        networkstatus_vote_free(v->vote);
+        tor_free(v);
+      });
+  } else {
+    /* Otherwise, we move them into "previous". */
+    smartlist_add_all(previous_vote_list, pending_vote_list);
+  }
+  smartlist_clear(pending_vote_list);
+
+  if (pending_consensus_signature_list) {
+    SMARTLIST_FOREACH(pending_consensus_signature_list, char *, cp,
+                      tor_free(cp));
+    smartlist_clear(pending_consensus_signature_list);
+  }
+  tor_free(pending_consensus_signatures);
+  dirvote_clear_pending_consensuses();
+}
+
+/** Return a newly allocated string containing the hex-encoded v3 authority
+    identity digest of every recognized v3 authority. */
+static char *
+list_v3_auth_ids(void)
+{
+  smartlist_t *known_v3_keys = smartlist_new();
+  char *keys;
+  SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
+                    dir_server_t *, ds,
+    if ((ds->type & V3_DIRINFO) &&
+        !tor_digest_is_zero(ds->v3_identity_digest))
+      smartlist_add(known_v3_keys,
+                    tor_strdup(hex_str(ds->v3_identity_digest, DIGEST_LEN))));
+  keys = smartlist_join_strings(known_v3_keys, ", ", 0, NULL);
+  SMARTLIST_FOREACH(known_v3_keys, char *, cp, tor_free(cp));
+  smartlist_free(known_v3_keys);
+  return keys;
+}
+
+/** Called when we have received a networkstatus vote in <b>vote_body</b>.
+ * Parse and validate it, and on success store it as a pending vote (which we
+ * then return).  Return NULL on failure.  Sets *<b>msg_out</b> and
+ * *<b>status_out</b> to an HTTP response and status code.  (V3 authority
+ * only) */
+pending_vote_t *
+dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
+{
+  networkstatus_t *vote;
+  networkstatus_voter_info_t *vi;
+  dir_server_t *ds;
+  pending_vote_t *pending_vote = NULL;
+  const char *end_of_vote = NULL;
+  int any_failed = 0;
+  tor_assert(vote_body);
+  tor_assert(msg_out);
+  tor_assert(status_out);
+
+  if (!pending_vote_list)
+    pending_vote_list = smartlist_new();
+  *status_out = 0;
+  *msg_out = NULL;
+
+ again:
+  vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote,
+                                              NS_TYPE_VOTE);
+  if (!end_of_vote)
+    end_of_vote = vote_body + strlen(vote_body);
+  if (!vote) {
+    log_warn(LD_DIR, "Couldn't parse vote: length was %d",
+             (int)strlen(vote_body));
+    *msg_out = "Unable to parse vote";
+    goto err;
+  }
+  tor_assert(smartlist_len(vote->voters) == 1);
+  vi = get_voter(vote);
+  {
+    int any_sig_good = 0;
+    SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig,
+                      if (sig->good_signature)
+                        any_sig_good = 1);
+    tor_assert(any_sig_good);
+  }
+  ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest);
+  if (!ds) {
+    char *keys = list_v3_auth_ids();
+    log_warn(LD_DIR, "Got a vote from an authority (nickname %s, address %s) "
+             "with authority key ID %s. "
+             "This key ID is not recognized.  Known v3 key IDs are: %s",
+             vi->nickname, vi->address,
+             hex_str(vi->identity_digest, DIGEST_LEN), keys);
+    tor_free(keys);
+    *msg_out = "Vote not from a recognized v3 authority";
+    goto err;
+  }
+  tor_assert(vote->cert);
+  if (!authority_cert_get_by_digests(vote->cert->cache_info.identity_digest,
+                                     vote->cert->signing_key_digest)) {
+    /* Hey, it's a new cert! */
+    trusted_dirs_load_certs_from_string(
+                               vote->cert->cache_info.signed_descriptor_body,
+                               TRUSTED_DIRS_CERTS_SRC_FROM_VOTE, 1 /*flush*/,
+                               NULL);
+    if (!authority_cert_get_by_digests(vote->cert->cache_info.identity_digest,
+                                       vote->cert->signing_key_digest)) {
+      log_warn(LD_BUG, "We added a cert, but still couldn't find it.");
+    }
+  }
+
+  /* Is it for the right period? */
+  if (vote->valid_after != voting_schedule.interval_starts) {
+    char tbuf1[ISO_TIME_LEN+1], tbuf2[ISO_TIME_LEN+1];
+    format_iso_time(tbuf1, vote->valid_after);
+    format_iso_time(tbuf2, voting_schedule.interval_starts);
+    log_warn(LD_DIR, "Rejecting vote from %s with valid-after time of %s; "
+             "we were expecting %s", vi->address, tbuf1, tbuf2);
+    *msg_out = "Bad valid-after time";
+    goto err;
+  }
+
+  /* Fetch any new router descriptors we just learned about */
+  update_consensus_router_descriptor_downloads(time(NULL), 1, vote);
+
+  /* Now see whether we already have a vote from this authority. */
+  SMARTLIST_FOREACH_BEGIN(pending_vote_list, pending_vote_t *, v) {
+      if (fast_memeq(v->vote->cert->cache_info.identity_digest,
+                   vote->cert->cache_info.identity_digest,
+                   DIGEST_LEN)) {
+        networkstatus_voter_info_t *vi_old = get_voter(v->vote);
+        if (fast_memeq(vi_old->vote_digest, vi->vote_digest, DIGEST_LEN)) {
+          /* Ah, it's the same vote. Not a problem. */
+          log_info(LD_DIR, "Discarding a vote we already have (from %s).",
+                   vi->address);
+          if (*status_out < 200)
+            *status_out = 200;
+          goto discard;
+        } else if (v->vote->published < vote->published) {
+          log_notice(LD_DIR, "Replacing an older pending vote from this "
+                     "directory (%s)", vi->address);
+          cached_dir_decref(v->vote_body);
+          networkstatus_vote_free(v->vote);
+          v->vote_body = new_cached_dir(tor_strndup(vote_body,
+                                                    end_of_vote-vote_body),
+                                        vote->published);
+          v->vote = vote;
+          if (end_of_vote &&
+              !strcmpstart(end_of_vote, "network-status-version"))
+            goto again;
+
+          if (*status_out < 200)
+            *status_out = 200;
+          if (!*msg_out)
+            *msg_out = "OK";
+          return v;
+        } else {
+          *msg_out = "Already have a newer pending vote";
+          goto err;
+        }
+      }
+  } SMARTLIST_FOREACH_END(v);
+
+  /* This a valid vote, update our shared random state. */
+  sr_handle_received_commits(vote->sr_info.commits,
+                             vote->cert->identity_key);
+
+  pending_vote = tor_malloc_zero(sizeof(pending_vote_t));
+  pending_vote->vote_body = new_cached_dir(tor_strndup(vote_body,
+                                                       end_of_vote-vote_body),
+                                           vote->published);
+  pending_vote->vote = vote;
+  smartlist_add(pending_vote_list, pending_vote);
+
+  if (!strcmpstart(end_of_vote, "network-status-version ")) {
+    vote_body = end_of_vote;
+    goto again;
+  }
+
+  goto done;
+
+ err:
+  any_failed = 1;
+  if (!*msg_out)
+    *msg_out = "Error adding vote";
+  if (*status_out < 400)
+    *status_out = 400;
+
+ discard:
+  networkstatus_vote_free(vote);
+
+  if (end_of_vote && !strcmpstart(end_of_vote, "network-status-version ")) {
+    vote_body = end_of_vote;
+    goto again;
+  }
+
+ done:
+
+  if (*status_out < 200)
+    *status_out = 200;
+  if (!*msg_out) {
+    if (!any_failed && !pending_vote) {
+      *msg_out = "Duplicate discarded";
+    } else {
+      *msg_out = "ok";
+    }
+  }
+
+  return any_failed ? NULL : pending_vote;
+}
+
+/* Write the votes in <b>pending_vote_list</b> to disk. */
+static void
+write_v3_votes_to_disk(const smartlist_t *pending_votes)
+{
+  smartlist_t *votestrings = smartlist_new();
+  char *votefile = NULL;
+
+  SMARTLIST_FOREACH(pending_votes, pending_vote_t *, v,
+    {
+      sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t));
+      c->bytes = v->vote_body->dir;
+      c->len = v->vote_body->dir_len;
+      smartlist_add(votestrings, c); /* collect strings to write to disk */
+    });
+
+  votefile = get_datadir_fname("v3-status-votes");
+  write_chunks_to_file(votefile, votestrings, 0, 0);
+  log_debug(LD_DIR, "Wrote votes to disk (%s)!", votefile);
+
+  tor_free(votefile);
+  SMARTLIST_FOREACH(votestrings, sized_chunk_t *, c, tor_free(c));
+  smartlist_free(votestrings);
+}
+
+/** Try to compute a v3 networkstatus consensus from the currently pending
+ * votes.  Return 0 on success, -1 on failure.  Store the consensus in
+ * pending_consensus: it won't be ready to be published until we have
+ * everybody else's signatures collected too. (V3 Authority only) */
+static int
+dirvote_compute_consensuses(void)
+{
+  /* Have we got enough votes to try? */
+  int n_votes, n_voters, n_vote_running = 0;
+  smartlist_t *votes = NULL;
+  char *consensus_body = NULL, *signatures = NULL;
+  networkstatus_t *consensus = NULL;
+  authority_cert_t *my_cert;
+  pending_consensus_t pending[N_CONSENSUS_FLAVORS];
+  int flav;
+
+  memset(pending, 0, sizeof(pending));
+
+  if (!pending_vote_list)
+    pending_vote_list = smartlist_new();
+
+  /* Write votes to disk */
+  write_v3_votes_to_disk(pending_vote_list);
+
+  /* Setup votes smartlist */
+  votes = smartlist_new();
+  SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v,
+    {
+      smartlist_add(votes, v->vote); /* collect votes to compute consensus */
+    });
+
+  /* See if consensus managed to achieve majority */
+  n_voters = get_n_authorities(V3_DIRINFO);
+  n_votes = smartlist_len(pending_vote_list);
+  if (n_votes <= n_voters/2) {
+    log_warn(LD_DIR, "We don't have enough votes to generate a consensus: "
+             "%d of %d", n_votes, n_voters/2+1);
+    goto err;
+  }
+  tor_assert(pending_vote_list);
+  SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, {
+    if (smartlist_contains_string(v->vote->known_flags, "Running"))
+      n_vote_running++;
+  });
+  if (!n_vote_running) {
+    /* See task 1066. */
+    log_warn(LD_DIR, "Nobody has voted on the Running flag. Generating "
+                     "and publishing a consensus without Running nodes "
+                     "would make many clients stop working. Not "
+                     "generating a consensus!");
+    goto err;
+  }
+
+  if (!(my_cert = get_my_v3_authority_cert())) {
+    log_warn(LD_DIR, "Can't generate consensus without a certificate.");
+    goto err;
+  }
+
+  {
+    char legacy_dbuf[DIGEST_LEN];
+    crypto_pk_t *legacy_sign=NULL;
+    char *legacy_id_digest = NULL;
+    int n_generated = 0;
+    if (get_options()->V3AuthUseLegacyKey) {
+      authority_cert_t *cert = get_my_v3_legacy_cert();
+      legacy_sign = get_my_v3_legacy_signing_key();
+      if (cert) {
+        if (crypto_pk_get_digest(cert->identity_key, legacy_dbuf)) {
+          log_warn(LD_BUG,
+                   "Unable to compute digest of legacy v3 identity key");
+        } else {
+          legacy_id_digest = legacy_dbuf;
+        }
+      }
+    }
+
+    for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+      const char *flavor_name = networkstatus_get_flavor_name(flav);
+      consensus_body = networkstatus_compute_consensus(
+        votes, n_voters,
+        my_cert->identity_key,
+        get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign,
+        flav);
+
+      if (!consensus_body) {
+        log_warn(LD_DIR, "Couldn't generate a %s consensus at all!",
+                 flavor_name);
+        continue;
+      }
+      consensus = networkstatus_parse_vote_from_string(consensus_body, NULL,
+                                                       NS_TYPE_CONSENSUS);
+      if (!consensus) {
+        log_warn(LD_DIR, "Couldn't parse %s consensus we generated!",
+                 flavor_name);
+        tor_free(consensus_body);
+        continue;
+      }
+
+      /* 'Check' our own signature, to mark it valid. */
+      networkstatus_check_consensus_signature(consensus, -1);
+
+      pending[flav].body = consensus_body;
+      pending[flav].consensus = consensus;
+      n_generated++;
+      consensus_body = NULL;
+      consensus = NULL;
+    }
+    if (!n_generated) {
+      log_warn(LD_DIR, "Couldn't generate any consensus flavors at all.");
+      goto err;
+    }
+  }
+
+  signatures = get_detached_signatures_from_pending_consensuses(
+       pending, N_CONSENSUS_FLAVORS);
+
+  if (!signatures) {
+    log_warn(LD_DIR, "Couldn't extract signatures.");
+    goto err;
+  }
+
+  dirvote_clear_pending_consensuses();
+  memcpy(pending_consensuses, pending, sizeof(pending));
+
+  tor_free(pending_consensus_signatures);
+  pending_consensus_signatures = signatures;
+
+  if (pending_consensus_signature_list) {
+    int n_sigs = 0;
+    /* we may have gotten signatures for this consensus before we built
+     * it ourself.  Add them now. */
+    SMARTLIST_FOREACH_BEGIN(pending_consensus_signature_list, char *, sig) {
+        const char *msg = NULL;
+        int r = dirvote_add_signatures_to_all_pending_consensuses(sig,
+                                                     "pending", &msg);
+        if (r >= 0)
+          n_sigs += r;
+        else
+          log_warn(LD_DIR,
+                   "Could not add queued signature to new consensus: %s",
+                   msg);
+        tor_free(sig);
+    } SMARTLIST_FOREACH_END(sig);
+    if (n_sigs)
+      log_notice(LD_DIR, "Added %d pending signatures while building "
+                 "consensus.", n_sigs);
+    smartlist_clear(pending_consensus_signature_list);
+  }
+
+  log_notice(LD_DIR, "Consensus computed; uploading signature(s)");
+
+  directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_SIGNATURES,
+                               ROUTER_PURPOSE_GENERAL,
+                               V3_DIRINFO,
+                               pending_consensus_signatures,
+                               strlen(pending_consensus_signatures), 0);
+  log_notice(LD_DIR, "Signature(s) posted.");
+
+  smartlist_free(votes);
+  return 0;
+ err:
+  smartlist_free(votes);
+  tor_free(consensus_body);
+  tor_free(signatures);
+  networkstatus_vote_free(consensus);
+
+  return -1;
+}
+
+/** Helper: we just got the <b>detached_signatures_body</b> sent to us as
+ * signatures on the currently pending consensus.  Add them to <b>pc</b>
+ * as appropriate.  Return the number of signatures added. (?) */
+static int
+dirvote_add_signatures_to_pending_consensus(
+                       pending_consensus_t *pc,
+                       ns_detached_signatures_t *sigs,
+                       const char *source,
+                       int severity,
+                       const char **msg_out)
+{
+  const char *flavor_name;
+  int r = -1;
+
+  /* Only call if we have a pending consensus right now. */
+  tor_assert(pc->consensus);
+  tor_assert(pc->body);
+  tor_assert(pending_consensus_signatures);
+
+  flavor_name = networkstatus_get_flavor_name(pc->consensus->flavor);
+  *msg_out = NULL;
+
+  {
+    smartlist_t *sig_list = strmap_get(sigs->signatures, flavor_name);
+    log_info(LD_DIR, "Have %d signatures for adding to %s consensus.",
+             sig_list ? smartlist_len(sig_list) : 0, flavor_name);
+  }
+  r = networkstatus_add_detached_signatures(pc->consensus, sigs,
+                                            source, severity, msg_out);
+  if (r >= 0) {
+    log_info(LD_DIR,"Added %d signatures to consensus.", r);
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_DIR,
+           "Unable to add signatures to consensus: %s",
+           *msg_out ? *msg_out : "(unknown)");
+  }
+
+  if (r >= 1) {
+    char *new_signatures =
+      networkstatus_format_signatures(pc->consensus, 0);
+    char *dst, *dst_end;
+    size_t new_consensus_len;
+    if (!new_signatures) {
+      *msg_out = "No signatures to add";
+      goto err;
+    }
+    new_consensus_len =
+      strlen(pc->body) + strlen(new_signatures) + 1;
+    pc->body = tor_realloc(pc->body, new_consensus_len);
+    dst_end = pc->body + new_consensus_len;
+    dst = strstr(pc->body, "directory-signature ");
+    tor_assert(dst);
+    strlcpy(dst, new_signatures, dst_end-dst);
+
+    /* We remove this block once it has failed to crash for a while.  But
+     * unless it shows up in profiles, we're probably better leaving it in,
+     * just in case we break detached signature processing at some point. */
+    {
+      networkstatus_t *v = networkstatus_parse_vote_from_string(
+                                             pc->body, NULL,
+                                             NS_TYPE_CONSENSUS);
+      tor_assert(v);
+      networkstatus_vote_free(v);
+    }
+    *msg_out = "Signatures added";
+    tor_free(new_signatures);
+  } else if (r == 0) {
+    *msg_out = "Signatures ignored";
+  } else {
+    goto err;
+  }
+
+  goto done;
+ err:
+  if (!*msg_out)
+    *msg_out = "Unrecognized error while adding detached signatures.";
+ done:
+  return r;
+}
+
+static int
+dirvote_add_signatures_to_all_pending_consensuses(
+                       const char *detached_signatures_body,
+                       const char *source,
+                       const char **msg_out)
+{
+  int r=0, i, n_added = 0, errors = 0;
+  ns_detached_signatures_t *sigs;
+  tor_assert(detached_signatures_body);
+  tor_assert(msg_out);
+  tor_assert(pending_consensus_signatures);
+
+  if (!(sigs = networkstatus_parse_detached_signatures(
+                               detached_signatures_body, NULL))) {
+    *msg_out = "Couldn't parse detached signatures.";
+    goto err;
+  }
+
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    int res;
+    int severity = i == FLAV_NS ? LOG_NOTICE : LOG_INFO;
+    pending_consensus_t *pc = &pending_consensuses[i];
+    if (!pc->consensus)
+      continue;
+    res = dirvote_add_signatures_to_pending_consensus(pc, sigs, source,
+                                                      severity, msg_out);
+    if (res < 0)
+      errors++;
+    else
+      n_added += res;
+  }
+
+  if (errors && !n_added) {
+    r = -1;
+    goto err;
+  }
+
+  if (n_added && pending_consensuses[FLAV_NS].consensus) {
+    char *new_detached =
+      get_detached_signatures_from_pending_consensuses(
+                      pending_consensuses, N_CONSENSUS_FLAVORS);
+    if (new_detached) {
+      tor_free(pending_consensus_signatures);
+      pending_consensus_signatures = new_detached;
+    }
+  }
+
+  r = n_added;
+  goto done;
+ err:
+  if (!*msg_out)
+    *msg_out = "Unrecognized error while adding detached signatures.";
+ done:
+  ns_detached_signatures_free(sigs);
+  /* XXXX NM Check how return is used.  We can now have an error *and*
+     signatures added. */
+  return r;
+}
+
+/** Helper: we just got the <b>detached_signatures_body</b> sent to us as
+ * signatures on the currently pending consensus.  Add them to the pending
+ * consensus (if we have one); otherwise queue them until we have a
+ * consensus.  Return negative on failure, nonnegative on success. */
+int
+dirvote_add_signatures(const char *detached_signatures_body,
+                       const char *source,
+                       const char **msg)
+{
+  if (pending_consensuses[FLAV_NS].consensus) {
+    log_notice(LD_DIR, "Got a signature from %s. "
+                       "Adding it to the pending consensus.", source);
+    return dirvote_add_signatures_to_all_pending_consensuses(
+                                     detached_signatures_body, source, msg);
+  } else {
+    log_notice(LD_DIR, "Got a signature from %s. "
+                       "Queuing it for the next consensus.", source);
+    if (!pending_consensus_signature_list)
+      pending_consensus_signature_list = smartlist_new();
+    smartlist_add_strdup(pending_consensus_signature_list,
+                  detached_signatures_body);
+    *msg = "Signature queued";
+    return 0;
+  }
+}
+
+/** Replace the consensus that we're currently serving with the one that we've
+ * been building. (V3 Authority only) */
+static int
+dirvote_publish_consensus(void)
+{
+  int i;
+
+  /* Now remember all the other consensuses as if we were a directory cache. */
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    pending_consensus_t *pending = &pending_consensuses[i];
+    const char *name;
+    name = networkstatus_get_flavor_name(i);
+    tor_assert(name);
+    if (!pending->consensus ||
+      networkstatus_check_consensus_signature(pending->consensus, 1)<0) {
+      log_warn(LD_DIR, "Not enough info to publish pending %s consensus",name);
+      continue;
+    }
+
+    if (networkstatus_set_current_consensus(pending->body, name, 0, NULL))
+      log_warn(LD_DIR, "Error publishing %s consensus", name);
+    else
+      log_notice(LD_DIR, "Published %s consensus", name);
+  }
+
+  return 0;
+}
+
+/** Release all static storage held in dirvote.c */
+void
+dirvote_free_all(void)
+{
+  dirvote_clear_votes(1);
+  /* now empty as a result of dirvote_clear_votes(). */
+  smartlist_free(pending_vote_list);
+  pending_vote_list = NULL;
+  smartlist_free(previous_vote_list);
+  previous_vote_list = NULL;
+
+  dirvote_clear_pending_consensuses();
+  tor_free(pending_consensus_signatures);
+  if (pending_consensus_signature_list) {
+    /* now empty as a result of dirvote_clear_votes(). */
+    smartlist_free(pending_consensus_signature_list);
+    pending_consensus_signature_list = NULL;
+  }
+}
+
+/* ====
+ * Access to pending items.
+ * ==== */
+
+/** Return the body of the consensus that we're currently trying to build. */
+MOCK_IMPL(const char *,
+dirvote_get_pending_consensus, (consensus_flavor_t flav))
+{
+  tor_assert(((int)flav) >= 0 && (int)flav < N_CONSENSUS_FLAVORS);
+  return pending_consensuses[flav].body;
+}
+
+/** Return the signatures that we know for the consensus that we're currently
+ * trying to build. */
+MOCK_IMPL(const char *,
+dirvote_get_pending_detached_signatures, (void))
+{
+  return pending_consensus_signatures;
+}
+
+/** Return a given vote specified by <b>fp</b>.  If <b>by_id</b>, return the
+ * vote for the authority with the v3 authority identity key digest <b>fp</b>;
+ * if <b>by_id</b> is false, return the vote whose digest is <b>fp</b>.  If
+ * <b>fp</b> is NULL, return our own vote.  If <b>include_previous</b> is
+ * false, do not consider any votes for a consensus that's already been built.
+ * If <b>include_pending</b> is false, do not consider any votes for the
+ * consensus that's in progress.  May return NULL if we have no vote for the
+ * authority in question. */
+const cached_dir_t *
+dirvote_get_vote(const char *fp, int flags)
+{
+  int by_id = flags & DGV_BY_ID;
+  const int include_pending = flags & DGV_INCLUDE_PENDING;
+  const int include_previous = flags & DGV_INCLUDE_PREVIOUS;
+
+  if (!pending_vote_list && !previous_vote_list)
+    return NULL;
+  if (fp == NULL) {
+    authority_cert_t *c = get_my_v3_authority_cert();
+    if (c) {
+      fp = c->cache_info.identity_digest;
+      by_id = 1;
+    } else
+      return NULL;
+  }
+  if (by_id) {
+    if (pending_vote_list && include_pending) {
+      SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv,
+        if (fast_memeq(get_voter(pv->vote)->identity_digest, fp, DIGEST_LEN))
+          return pv->vote_body);
+    }
+    if (previous_vote_list && include_previous) {
+      SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv,
+        if (fast_memeq(get_voter(pv->vote)->identity_digest, fp, DIGEST_LEN))
+          return pv->vote_body);
+    }
+  } else {
+    if (pending_vote_list && include_pending) {
+      SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv,
+        if (fast_memeq(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN))
+          return pv->vote_body);
+    }
+    if (previous_vote_list && include_previous) {
+      SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv,
+        if (fast_memeq(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN))
+          return pv->vote_body);
+    }
+  }
+  return NULL;
+}
+
+/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>
+ * according to <b>consensus_method</b>.
+ **/
+STATIC microdesc_t *
+dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
+{
+  microdesc_t *result = NULL;
+  char *key = NULL, *summary = NULL, *family = NULL;
+  size_t keylen;
+  smartlist_t *chunks = smartlist_new();
+  char *output = NULL;
+
+  if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0)
+    goto done;
+  summary = policy_summarize(ri->exit_policy, AF_INET);
+  if (ri->declared_family)
+    family = smartlist_join_strings(ri->declared_family, " ", 0, NULL);
+
+  smartlist_add_asprintf(chunks, "onion-key\n%s", key);
+
+  if (ri->onion_curve25519_pkey) {
+    char kbuf[128];
+    base64_encode(kbuf, sizeof(kbuf),
+                  (const char*)ri->onion_curve25519_pkey->public_key,
+                  CURVE25519_PUBKEY_LEN, BASE64_ENCODE_MULTILINE);
+    smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf);
+  }
+
+  /* We originally put a lines in the micrdescriptors, but then we worked out
+   * that we needed them in the microdesc consensus. See #20916. */
+  if (consensus_method < MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC &&
+      !tor_addr_is_null(&ri->ipv6_addr) && ri->ipv6_orport)
+    smartlist_add_asprintf(chunks, "a %s\n",
+                           fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
+
+  if (family)
+    smartlist_add_asprintf(chunks, "family %s\n", family);
+
+  if (summary && strcmp(summary, "reject 1-65535"))
+    smartlist_add_asprintf(chunks, "p %s\n", summary);
+
+  if (ri->ipv6_exit_policy) {
+    /* XXXX+++ This doesn't match proposal 208, which says these should
+     * be taken unchanged from the routerinfo.  That's bogosity, IMO:
+     * the proposal should have said to do this instead.*/
+    char *p6 = write_short_policy(ri->ipv6_exit_policy);
+    if (p6 && strcmp(p6, "reject 1-65535"))
+      smartlist_add_asprintf(chunks, "p6 %s\n", p6);
+    tor_free(p6);
+  }
+
+  {
+    char idbuf[ED25519_BASE64_LEN+1];
+    const char *keytype;
+    if (ri->cache_info.signing_key_cert &&
+        ri->cache_info.signing_key_cert->signing_key_included) {
+      keytype = "ed25519";
+      ed25519_public_to_base64(idbuf,
+                               &ri->cache_info.signing_key_cert->signing_key);
+    } else {
+      keytype = "rsa1024";
+      digest_to_base64(idbuf, ri->cache_info.identity_digest);
+    }
+    smartlist_add_asprintf(chunks, "id %s %s\n", keytype, idbuf);
+  }
+
+  output = smartlist_join_strings(chunks, "", 0, NULL);
+
+  {
+    smartlist_t *lst = microdescs_parse_from_string(output,
+                                                    output+strlen(output), 0,
+                                                    SAVED_NOWHERE, NULL);
+    if (smartlist_len(lst) != 1) {
+      log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse.");
+      SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md));
+      smartlist_free(lst);
+      goto done;
+    }
+    result = smartlist_get(lst, 0);
+    smartlist_free(lst);
+  }
+
+ done:
+  tor_free(output);
+  tor_free(key);
+  tor_free(summary);
+  tor_free(family);
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+    smartlist_free(chunks);
+  }
+  return result;
+}
+
+/** Format the appropriate vote line to describe the microdescriptor <b>md</b>
+ * in a consensus vote document.  Write it into the <b>out_len</b>-byte buffer
+ * in <b>out</b>.  Return -1 on failure and the number of characters written
+ * on success. */
+static ssize_t
+dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len,
+                                   const microdesc_t *md,
+                                   int consensus_method_low,
+                                   int consensus_method_high)
+{
+  ssize_t ret = -1;
+  char d64[BASE64_DIGEST256_LEN+1];
+  char *microdesc_consensus_methods =
+    make_consensus_method_list(consensus_method_low,
+                               consensus_method_high,
+                               ",");
+  tor_assert(microdesc_consensus_methods);
+
+  if (digest256_to_base64(d64, md->digest)<0)
+    goto out;
+
+  if (tor_snprintf(out_buf, out_buf_len, "m %s sha256=%s\n",
+                   microdesc_consensus_methods, d64)<0)
+    goto out;
+
+  ret = strlen(out_buf);
+
+ out:
+  tor_free(microdesc_consensus_methods);
+  return ret;
+}
+
+/** Array of start and end of consensus methods used for supported
+    microdescriptor formats. */
+static const struct consensus_method_range_t {
+  int low;
+  int high;
+} microdesc_consensus_methods[] = {
+  {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1},
+  {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD},
+  {-1, -1}
+};
+
+/** Helper type used when generating the microdescriptor lines in a directory
+ * vote. */
+typedef struct microdesc_vote_line_t {
+  int low;
+  int high;
+  microdesc_t *md;
+  struct microdesc_vote_line_t *next;
+} microdesc_vote_line_t;
+
+/** Generate and return a linked list of all the lines that should appear to
+ * describe a router's microdescriptor versions in a directory vote.
+ * Add the generated microdescriptors to <b>microdescriptors_out</b>. */
+vote_microdesc_hash_t *
+dirvote_format_all_microdesc_vote_lines(const routerinfo_t *ri, time_t now,
+                                        smartlist_t *microdescriptors_out)
+{
+  const struct consensus_method_range_t *cmr;
+  microdesc_vote_line_t *entries = NULL, *ep;
+  vote_microdesc_hash_t *result = NULL;
+
+  /* Generate the microdescriptors. */
+  for (cmr = microdesc_consensus_methods;
+       cmr->low != -1 && cmr->high != -1;
+       cmr++) {
+    microdesc_t *md = dirvote_create_microdescriptor(ri, cmr->low);
+    if (md) {
+      microdesc_vote_line_t *e =
+        tor_malloc_zero(sizeof(microdesc_vote_line_t));
+      e->md = md;
+      e->low = cmr->low;
+      e->high = cmr->high;
+      e->next = entries;
+      entries = e;
+    }
+  }
+
+  /* Compress adjacent identical ones */
+  for (ep = entries; ep; ep = ep->next) {
+    while (ep->next &&
+           fast_memeq(ep->md->digest, ep->next->md->digest, DIGEST256_LEN) &&
+           ep->low == ep->next->high + 1) {
+      microdesc_vote_line_t *next = ep->next;
+      ep->low = next->low;
+      microdesc_free(next->md);
+      ep->next = next->next;
+      tor_free(next);
+    }
+  }
+
+  /* Format them into vote_microdesc_hash_t, and add to microdescriptors_out.*/
+  while ((ep = entries)) {
+    char buf[128];
+    vote_microdesc_hash_t *h;
+    if (dirvote_format_microdesc_vote_line(buf, sizeof(buf), ep->md,
+                                           ep->low, ep->high) >= 0) {
+      h = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
+      h->microdesc_hash_line = tor_strdup(buf);
+      h->next = result;
+      result = h;
+      ep->md->last_listed = now;
+      smartlist_add(microdescriptors_out, ep->md);
+    }
+    entries = ep->next;
+    tor_free(ep);
+  }
+
+  return result;
+}
+
+/** Parse and extract all SR commits from <b>tokens</b> and place them in
+ *  <b>ns</b>. */
+static void
+extract_shared_random_commits(networkstatus_t *ns, const smartlist_t *tokens)
+{
+  smartlist_t *chunks = NULL;
+
+  tor_assert(ns);
+  tor_assert(tokens);
+  /* Commits are only present in a vote. */
+  tor_assert(ns->type == NS_TYPE_VOTE);
+
+  ns->sr_info.commits = smartlist_new();
+
+  smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT);
+  /* It's normal that a vote might contain no commits even if it participates
+   * in the SR protocol. Don't treat it as an error. */
+  if (commits == NULL) {
+    goto end;
+  }
+
+  /* Parse the commit. We do NO validation of number of arguments or ordering
+   * for forward compatibility, it's the parse commit job to inform us if it's
+   * supported or not. */
+  chunks = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) {
+    /* Extract all arguments and put them in the chunks list. */
+    for (int i = 0; i < tok->n_args; i++) {
+      smartlist_add(chunks, tok->args[i]);
+    }
+    sr_commit_t *commit = sr_parse_commit(chunks);
+    smartlist_clear(chunks);
+    if (commit == NULL) {
+      /* Get voter identity so we can warn that this dirauth vote contains
+       * commit we can't parse. */
+      networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0);
+      tor_assert(voter);
+      log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.",
+               escaped(tok->object_body),
+               hex_str(voter->identity_digest,
+                       sizeof(voter->identity_digest)));
+      /* Commitment couldn't be parsed. Continue onto the next commit because
+       * this one could be unsupported for instance. */
+      continue;
+    }
+    /* Add newly created commit object to the vote. */
+    smartlist_add(ns->sr_info.commits, commit);
+  } SMARTLIST_FOREACH_END(tok);
+
+ end:
+  smartlist_free(chunks);
+  smartlist_free(commits);
+}
+
+/* Using the given directory tokens in tokens, parse the shared random commits
+ * and put them in the given vote document ns.
+ *
+ * This also sets the SR participation flag if present in the vote. */
+void
+dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens)
+{
+  /* Does this authority participates in the SR protocol? */
+  directory_token_t *tok = find_opt_by_keyword(tokens, K_SR_FLAG);
+  if (tok) {
+    ns->sr_info.participate = 1;
+    /* Get the SR commitments and reveals from the vote. */
+    extract_shared_random_commits(ns, tokens);
+  }
+}
+
+/* For the given vote, free the shared random commits if any. */
+void
+dirvote_clear_commits(networkstatus_t *ns)
+{
+  tor_assert(ns->type == NS_TYPE_VOTE);
+
+  if (ns->sr_info.commits) {
+    SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c,
+                      sr_commit_free(c));
+    smartlist_free(ns->sr_info.commits);
+  }
+}
+
+/* The given url is the /tor/status-vote GET directory request. Populates the
+ * items list with strings that we can compress on the fly and dir_items with
+ * cached_dir_t objects that have a precompressed deflated version. */
+void
+dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
+                               smartlist_t *dir_items)
+{
+  int current;
+
+  url += strlen("/tor/status-vote/");
+  current = !strcmpstart(url, "current/");
+  url = strchr(url, '/');
+  tor_assert(url);
+  ++url;
+  if (!strcmp(url, "consensus")) {
+    const char *item;
+    tor_assert(!current); /* we handle current consensus specially above,
+                           * since it wants to be spooled. */
+    if ((item = dirvote_get_pending_consensus(FLAV_NS)))
+      smartlist_add(items, (char*)item);
+  } else if (!current && !strcmp(url, "consensus-signatures")) {
+    /* XXXX the spec says that we should implement
+     * current/consensus-signatures too.  It doesn't seem to be needed,
+     * though. */
+    const char *item;
+    if ((item=dirvote_get_pending_detached_signatures()))
+      smartlist_add(items, (char*)item);
+  } else if (!strcmp(url, "authority")) {
+    const cached_dir_t *d;
+    int flags = DGV_BY_ID |
+      (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
+    if ((d=dirvote_get_vote(NULL, flags)))
+      smartlist_add(dir_items, (cached_dir_t*)d);
+  } else {
+    const cached_dir_t *d;
+    smartlist_t *fps = smartlist_new();
+    int flags;
+    if (!strcmpstart(url, "d/")) {
+      url += 2;
+      flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS;
+    } else {
+      flags = DGV_BY_ID |
+        (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
+    }
+    dir_split_resource_into_fingerprints(url, fps, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH(fps, char *, fp, {
+                      if ((d = dirvote_get_vote(fp, flags)))
+                      smartlist_add(dir_items, (cached_dir_t*)d);
+                      tor_free(fp);
+                      });
+    smartlist_free(fps);
+  }
+}
+
+/** Get the best estimate of a router's bandwidth for dirauth purposes,
+ * preferring measured to advertised values if available. */
+static uint32_t
+dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri)
+{
+  uint32_t bw_kb = 0;
+  /*
+   * Yeah, measured bandwidths in measured_bw_line_t are (implicitly
+   * signed) longs and the ones router_get_advertised_bandwidth() returns
+   * are uint32_t.
+   */
+  long mbw_kb = 0;
+
+  if (ri) {
+    /*
+     * * First try to see if we have a measured bandwidth; don't bother with
+     * as_of_out here, on the theory that a stale measured bandwidth is still
+     * better to trust than an advertised one.
+     */
+    if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
+                                           &mbw_kb, NULL)) {
+      /* Got one! */
+      bw_kb = (uint32_t)mbw_kb;
+    } else {
+      /* If not, fall back to advertised */
+      bw_kb = router_get_advertised_bandwidth(ri) / 1000;
+    }
+  }
+
+  return bw_kb;
+}
+
+/** Helper for sorting: compares two routerinfos first by address, and then by
+ * descending order of "usefulness".  (An authority is more useful than a
+ * non-authority; a running router is more useful than a non-running router;
+ * and a router with more bandwidth is more useful than one with less.)
+ **/
+static int
+compare_routerinfo_by_ip_and_bw_(const void **a, const void **b)
+{
+  routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b;
+  int first_is_auth, second_is_auth;
+  uint32_t bw_kb_first, bw_kb_second;
+  const node_t *node_first, *node_second;
+  int first_is_running, second_is_running;
+
+  /* we return -1 if first should appear before second... that is,
+   * if first is a better router. */
+  if (first->addr < second->addr)
+    return -1;
+  else if (first->addr > second->addr)
+    return 1;
+
+  /* Potentially, this next bit could cause k n lg n memeq calls.  But in
+   * reality, we will almost never get here, since addresses will usually be
+   * different. */
+
+  first_is_auth =
+    router_digest_is_trusted_dir(first->cache_info.identity_digest);
+  second_is_auth =
+    router_digest_is_trusted_dir(second->cache_info.identity_digest);
+
+  if (first_is_auth && !second_is_auth)
+    return -1;
+  else if (!first_is_auth && second_is_auth)
+    return 1;
+
+  node_first = node_get_by_id(first->cache_info.identity_digest);
+  node_second = node_get_by_id(second->cache_info.identity_digest);
+  first_is_running = node_first && node_first->is_running;
+  second_is_running = node_second && node_second->is_running;
+
+  if (first_is_running && !second_is_running)
+    return -1;
+  else if (!first_is_running && second_is_running)
+    return 1;
+
+  bw_kb_first = dirserv_get_bandwidth_for_router_kb(first);
+  bw_kb_second = dirserv_get_bandwidth_for_router_kb(second);
+
+  if (bw_kb_first > bw_kb_second)
+    return -1;
+  else if (bw_kb_first < bw_kb_second)
+    return 1;
+
+  /* They're equal! Compare by identity digest, so there's a
+   * deterministic order and we avoid flapping. */
+  return fast_memcmp(first->cache_info.identity_digest,
+                     second->cache_info.identity_digest,
+                     DIGEST_LEN);
+}
+
+/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t
+ * whose keys are the identity digests of those routers that we're going to
+ * exclude for Sybil-like appearance. */
+static digestmap_t *
+get_possible_sybil_list(const smartlist_t *routers)
+{
+  const or_options_t *options = get_options();
+  digestmap_t *omit_as_sybil;
+  smartlist_t *routers_by_ip = smartlist_new();
+  uint32_t last_addr;
+  int addr_count;
+  /* Allow at most this number of Tor servers on a single IP address, ... */
+  int max_with_same_addr = options->AuthDirMaxServersPerAddr;
+  if (max_with_same_addr <= 0)
+    max_with_same_addr = INT_MAX;
+
+  smartlist_add_all(routers_by_ip, routers);
+  smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_);
+  omit_as_sybil = digestmap_new();
+
+  last_addr = 0;
+  addr_count = 0;
+  SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) {
+    if (last_addr != ri->addr) {
+      last_addr = ri->addr;
+      addr_count = 1;
+    } else if (++addr_count > max_with_same_addr) {
+      digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  smartlist_free(routers_by_ip);
+  return omit_as_sybil;
+}
+
+/** Given a platform string as in a routerinfo_t (possibly null), return a
+ * newly allocated version string for a networkstatus document, or NULL if the
+ * platform doesn't give a Tor version. */
+static char *
+version_from_platform(const char *platform)
+{
+  if (platform && !strcmpstart(platform, "Tor ")) {
+    const char *eos = find_whitespace(platform+4);
+    if (eos && !strcmpstart(eos, " (r")) {
+      /* XXXX Unify this logic with the other version extraction
+       * logic in routerparse.c. */
+      eos = find_whitespace(eos+1);
+    }
+    if (eos) {
+      return tor_strndup(platform, eos-platform);
+    }
+  }
+  return NULL;
+}
+
+/** Given a (possibly empty) list of config_line_t, each line of which contains
+ * a list of comma-separated version numbers surrounded by optional space,
+ * allocate and return a new string containing the version numbers, in order,
+ * separated by commas.  Used to generate Recommended(Client|Server)?Versions
+ */
+static char *
+format_versions_list(config_line_t *ln)
+{
+  smartlist_t *versions;
+  char *result;
+  versions = smartlist_new();
+  for ( ; ln; ln = ln->next) {
+    smartlist_split_string(versions, ln->value, ",",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  }
+  sort_version_list(versions, 1);
+  result = smartlist_join_strings(versions,",",0,NULL);
+  SMARTLIST_FOREACH(versions,char *,s,tor_free(s));
+  smartlist_free(versions);
+  return result;
+}
+
+/** If there are entries in <b>routers</b> with exactly the same ed25519 keys,
+ * remove the older one.  If they are exactly the same age, remove the one
+ * with the greater descriptor digest. May alter the order of the list. */
+static void
+routers_make_ed_keys_unique(smartlist_t *routers)
+{
+  routerinfo_t *ri2;
+  digest256map_t *by_ed_key = digest256map_new();
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    ri->omit_from_vote = 0;
+    if (ri->cache_info.signing_key_cert == NULL)
+      continue; /* No ed key */
+    const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey;
+    if ((ri2 = digest256map_get(by_ed_key, pk))) {
+      /* Duplicate; must omit one.  Set the omit_from_vote flag in whichever
+       * one has the earlier published_on. */
+      const time_t ri_pub = ri->cache_info.published_on;
+      const time_t ri2_pub = ri2->cache_info.published_on;
+      if (ri2_pub < ri_pub ||
+          (ri2_pub == ri_pub &&
+           fast_memcmp(ri->cache_info.signed_descriptor_digest,
+                     ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) {
+        digest256map_set(by_ed_key, pk, ri);
+        ri2->omit_from_vote = 1;
+      } else {
+        ri->omit_from_vote = 1;
+      }
+    } else {
+      /* Add to map */
+      digest256map_set(by_ed_key, pk, ri);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  digest256map_free(by_ed_key, NULL);
+
+  /* Now remove every router where the omit_from_vote flag got set. */
+  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
+    if (ri->omit_from_vote) {
+      SMARTLIST_DEL_CURRENT(routers, ri);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+}
+
+/** Routerstatus <b>rs</b> is part of a group of routers that are on
+ * too narrow an IP-space. Clear out its flags since we don't want it be used
+ * because of its Sybil-like appearance.
+ *
+ * Leave its BadExit flag alone though, since if we think it's a bad exit,
+ * we want to vote that way in case all the other authorities are voting
+ * Running and Exit.
+ */
+static void
+clear_status_flags_on_sybil(routerstatus_t *rs)
+{
+  rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast =
+    rs->is_flagged_running = rs->is_named = rs->is_valid =
+    rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0;
+  /* FFFF we might want some mechanism to check later on if we
+   * missed zeroing any flags: it's easy to add a new flag but
+   * forget to add it to this clause. */
+}
+
+/** Return a new networkstatus_t* containing our current opinion. (For v3
+ * authorities) */
+networkstatus_t *
+dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
+                                        authority_cert_t *cert)
+{
+  const or_options_t *options = get_options();
+  networkstatus_t *v3_out = NULL;
+  uint32_t addr;
+  char *hostname = NULL, *client_versions = NULL, *server_versions = NULL;
+  const char *contact;
+  smartlist_t *routers, *routerstatuses;
+  char identity_digest[DIGEST_LEN];
+  char signing_key_digest[DIGEST_LEN];
+  int listbadexits = options->AuthDirListBadExits;
+  routerlist_t *rl = router_get_routerlist();
+  time_t now = time(NULL);
+  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  networkstatus_voter_info_t *voter = NULL;
+  vote_timing_t timing;
+  digestmap_t *omit_as_sybil = NULL;
+  const int vote_on_reachability = running_long_enough_to_decide_unreachable();
+  smartlist_t *microdescriptors = NULL;
+
+  tor_assert(private_key);
+  tor_assert(cert);
+
+  if (crypto_pk_get_digest(private_key, signing_key_digest)<0) {
+    log_err(LD_BUG, "Error computing signing key digest");
+    return NULL;
+  }
+  if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) {
+    log_err(LD_BUG, "Error computing identity key digest");
+    return NULL;
+  }
+  if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) {
+    log_warn(LD_NET, "Couldn't resolve my hostname");
+    return NULL;
+  }
+  if (!hostname || !strchr(hostname, '.')) {
+    tor_free(hostname);
+    hostname = tor_dup_ip(addr);
+  }
+
+  if (options->VersioningAuthoritativeDir) {
+    client_versions = format_versions_list(options->RecommendedClientVersions);
+    server_versions = format_versions_list(options->RecommendedServerVersions);
+  }
+
+  contact = get_options()->ContactInfo;
+  if (!contact)
+    contact = "(none)";
+
+  /*
+   * Do this so dirserv_compute_performance_thresholds() and
+   * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
+   */
+  if (options->V3BandwidthsFile) {
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL);
+  } else {
+    /*
+     * No bandwidths file; clear the measured bandwidth cache in case we had
+     * one last time around.
+     */
+    if (dirserv_get_measured_bw_cache_size() > 0) {
+      dirserv_clear_measured_bw_cache();
+    }
+  }
+
+  /* precompute this part, since we need it to decide what "stable"
+   * means. */
+  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, {
+                    dirserv_set_router_is_running(ri, now);
+                    });
+
+  routers = smartlist_new();
+  smartlist_add_all(routers, rl->routers);
+  routers_make_ed_keys_unique(routers);
+  /* After this point, don't use rl->routers; use 'routers' instead. */
+  routers_sort_by_identity(routers);
+  omit_as_sybil = get_possible_sybil_list(routers);
+
+  DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) {
+    (void) ignore;
+    rep_hist_make_router_pessimal(sybil_id, now);
+  } DIGESTMAP_FOREACH_END;
+
+  /* Count how many have measured bandwidths so we know how to assign flags;
+   * this must come before dirserv_compute_performance_thresholds() */
+  dirserv_count_measured_bws(routers);
+
+  dirserv_compute_performance_thresholds(omit_as_sybil);
+
+  routerstatuses = smartlist_new();
+  microdescriptors = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    /* If it has a protover list and contains a protocol name greater than
+     * MAX_PROTOCOL_NAME_LENGTH, skip it. */
+    if (ri->protocol_list &&
+        protover_contains_long_protocol_names(ri->protocol_list)) {
+      continue;
+    }
+    if (ri->cache_info.published_on >= cutoff) {
+      routerstatus_t *rs;
+      vote_routerstatus_t *vrs;
+      node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+      if (!node)
+        continue;
+
+      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      rs = &vrs->status;
+      set_routerstatus_from_routerinfo(rs, node, ri, now,
+                                       listbadexits);
+
+      if (ri->cache_info.signing_key_cert) {
+        memcpy(vrs->ed25519_id,
+               ri->cache_info.signing_key_cert->signing_key.pubkey,
+               ED25519_PUBKEY_LEN);
+      }
+
+      if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest))
+        clear_status_flags_on_sybil(rs);
+
+      if (!vote_on_reachability)
+        rs->is_flagged_running = 0;
+
+      vrs->version = version_from_platform(ri->platform);
+      if (ri->protocol_list) {
+        vrs->protocols = tor_strdup(ri->protocol_list);
+      } else {
+        vrs->protocols = tor_strdup(
+                                protover_compute_for_old_tor(vrs->version));
+      }
+      vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now,
+                                                            microdescriptors);
+
+      smartlist_add(routerstatuses, vrs);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  {
+    smartlist_t *added =
+      microdescs_add_list_to_cache(get_microdesc_cache(),
+                                   microdescriptors, SAVED_NOWHERE, 0);
+    smartlist_free(added);
+    smartlist_free(microdescriptors);
+  }
+
+  smartlist_free(routers);
+  digestmap_free(omit_as_sybil, NULL);
+
+  /* Apply guardfraction information to routerstatuses. */
+  if (options->GuardfractionFile) {
+    dirserv_read_guardfraction_file(options->GuardfractionFile,
+                                    routerstatuses);
+  }
+
+  /* This pass through applies the measured bw lines to the routerstatuses */
+  if (options->V3BandwidthsFile) {
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
+                                     routerstatuses);
+  } else {
+    /*
+     * No bandwidths file; clear the measured bandwidth cache in case we had
+     * one last time around.
+     */
+    if (dirserv_get_measured_bw_cache_size() > 0) {
+      dirserv_clear_measured_bw_cache();
+    }
+  }
+
+  v3_out = tor_malloc_zero(sizeof(networkstatus_t));
+
+  v3_out->type = NS_TYPE_VOTE;
+  dirvote_get_preferred_voting_intervals(&timing);
+  v3_out->published = now;
+  {
+    char tbuf[ISO_TIME_LEN+1];
+    networkstatus_t *current_consensus =
+      networkstatus_get_live_consensus(now);
+    long last_consensus_interval; /* only used to pick a valid_after */
+    if (current_consensus)
+      last_consensus_interval = current_consensus->fresh_until -
+        current_consensus->valid_after;
+    else
+      last_consensus_interval = options->TestingV3AuthInitialVotingInterval;
+    v3_out->valid_after =
+      voting_schedule_get_start_of_next_interval(now,
+                                   (int)last_consensus_interval,
+                                   options->TestingV3AuthVotingStartOffset);
+    format_iso_time(tbuf, v3_out->valid_after);
+    log_notice(LD_DIR,"Choosing valid-after time in vote as %s: "
+               "consensus_set=%d, last_interval=%d",
+               tbuf, current_consensus?1:0, (int)last_consensus_interval);
+  }
+  v3_out->fresh_until = v3_out->valid_after + timing.vote_interval;
+  v3_out->valid_until = v3_out->valid_after +
+    (timing.vote_interval * timing.n_intervals_valid);
+  v3_out->vote_seconds = timing.vote_delay;
+  v3_out->dist_seconds = timing.dist_delay;
+  tor_assert(v3_out->vote_seconds > 0);
+  tor_assert(v3_out->dist_seconds > 0);
+  tor_assert(timing.n_intervals_valid > 0);
+
+  v3_out->client_versions = client_versions;
+  v3_out->server_versions = server_versions;
+
+  /* These are hardwired, to avoid disaster. */
+  v3_out->recommended_relay_protocols =
+    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2");
+  v3_out->recommended_client_protocols =
+    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2");
+  v3_out->required_client_protocols =
+    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2");
+  v3_out->required_relay_protocols =
+    tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2");
+
+  /* We are not allowed to vote to require anything we don't have. */
+  tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL));
+  tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL));
+
+  /* We should not recommend anything we don't have. */
+  tor_assert_nonfatal(protover_all_supported(
+                               v3_out->recommended_relay_protocols, NULL));
+  tor_assert_nonfatal(protover_all_supported(
+                               v3_out->recommended_client_protocols, NULL));
+
+  v3_out->package_lines = smartlist_new();
+  {
+    config_line_t *cl;
+    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
+      if (validate_recommended_package_line(cl->value))
+        smartlist_add_strdup(v3_out->package_lines, cl->value);
+    }
+  }
+
+  v3_out->known_flags = smartlist_new();
+  smartlist_split_string(v3_out->known_flags,
+                         "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
+                         0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  if (vote_on_reachability)
+    smartlist_add_strdup(v3_out->known_flags, "Running");
+  if (listbadexits)
+    smartlist_add_strdup(v3_out->known_flags, "BadExit");
+  smartlist_sort_strings(v3_out->known_flags);
+
+  if (options->ConsensusParams) {
+    v3_out->net_params = smartlist_new();
+    smartlist_split_string(v3_out->net_params,
+                           options->ConsensusParams, NULL, 0, 0);
+    smartlist_sort_strings(v3_out->net_params);
+  }
+
+  voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
+  voter->nickname = tor_strdup(options->Nickname);
+  memcpy(voter->identity_digest, identity_digest, DIGEST_LEN);
+  voter->sigs = smartlist_new();
+  voter->address = hostname;
+  voter->addr = addr;
+  voter->dir_port = router_get_advertised_dir_port(options, 0);
+  voter->or_port = router_get_advertised_or_port(options);
+  voter->contact = tor_strdup(contact);
+  if (options->V3AuthUseLegacyKey) {
+    authority_cert_t *c = get_my_v3_legacy_cert();
+    if (c) {
+      if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) {
+        log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key");
+        memset(voter->legacy_id_digest, 0, DIGEST_LEN);
+      }
+    }
+  }
+
+  v3_out->voters = smartlist_new();
+  smartlist_add(v3_out->voters, voter);
+  v3_out->cert = authority_cert_dup(cert);
+  v3_out->routerstatus_list = routerstatuses;
+  /* Note: networkstatus_digest is unset; it won't get set until we actually
+   * format the vote. */
+
+  return v3_out;
+}
diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h
new file mode 100644
index 0000000000..7ce8e4a699
--- /dev/null
+++ b/src/feature/dirauth/dirvote.h
@@ -0,0 +1,247 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirvote.h
+ * \brief Header file for dirvote.c.
+ **/
+
+#ifndef TOR_DIRVOTE_H
+#define TOR_DIRVOTE_H
+
+/*
+ * Ideally, assuming synced clocks, we should only need 1 second for each of:
+ *  - Vote
+ *  - Distribute
+ *  - Consensus Publication
+ * As we can gather descriptors continuously.
+ * (Could we even go as far as publishing the previous consensus,
+ *  in the same second that we vote for the next one?)
+ * But we're not there yet: these are the lowest working values at this time.
+ */
+
+/** Lowest allowable value for VoteSeconds. */
+#define MIN_VOTE_SECONDS 2
+/** Lowest allowable value for VoteSeconds when TestingTorNetwork is 1 */
+#define MIN_VOTE_SECONDS_TESTING 2
+
+/** Lowest allowable value for DistSeconds. */
+#define MIN_DIST_SECONDS 2
+/** Lowest allowable value for DistSeconds when TestingTorNetwork is 1 */
+#define MIN_DIST_SECONDS_TESTING 2
+
+/** Lowest allowable voting interval. */
+#define MIN_VOTE_INTERVAL 300
+/** Lowest allowable voting interval when TestingTorNetwork is 1:
+ * Voting Interval can be:
+ *   10, 12, 15, 18, 20, 24, 25, 30, 36, 40, 45, 50, 60, ...
+ * Testing Initial Voting Interval can be:
+ *    5,  6,  8,  9, or any of the possible values for Voting Interval,
+ * as they both need to evenly divide 30 minutes.
+ * If clock desynchronisation is an issue, use an interval of at least:
+ *   18 * drift in seconds, to allow for a clock slop factor */
+#define MIN_VOTE_INTERVAL_TESTING \
+                (((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)*2)
+
+#define MIN_VOTE_INTERVAL_TESTING_INITIAL \
+                ((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)
+
+/* A placeholder for routerstatus_format_entry() when the consensus method
+ * argument is not applicable. */
+#define ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD 0
+
+/** The lowest consensus method that we currently support. */
+#define MIN_SUPPORTED_CONSENSUS_METHOD 25
+
+/** The highest consensus method that we currently support. */
+#define MAX_SUPPORTED_CONSENSUS_METHOD 28
+
+/** Lowest consensus method where authorities vote on required/recommended
+ * protocols. */
+#define MIN_METHOD_FOR_RECOMMENDED_PROTOCOLS 25
+
+/** Lowest consensus method where authorities add protocols to routerstatus
+ * entries. */
+#define MIN_METHOD_FOR_RS_PROTOCOLS 25
+
+/** Lowest consensus method where authorities initialize bandwidth weights to 1
+ * instead of 0. See #14881 */
+#define MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE 26
+
+/** Lowest consensus method where the microdesc consensus contains relay IPv6
+ * addresses. See #23826 and #20916. */
+#define MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS 27
+
+/** Lowest consensus method where microdescriptors do not contain relay IPv6
+ * addresses. See #23828 and #20916. */
+#define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28
+
+/** Default bandwidth to clip unmeasured bandwidths to using method >=
+ * MIN_METHOD_TO_CLIP_UNMEASURED_BW.  (This is not a consensus method; do not
+ * get confused with the above macros.) */
+#define DEFAULT_MAX_UNMEASURED_BW_KB 20
+
+/* Directory Get Vote (DGV) flags for dirvote_get_vote(). */
+#define DGV_BY_ID 1
+#define DGV_INCLUDE_PENDING 2
+#define DGV_INCLUDE_PREVIOUS 4
+
+/*
+ * Public API. Used outside of the dirauth subsystem.
+ *
+ * We need to nullify them if the module is disabled.
+ */
+#ifdef HAVE_MODULE_DIRAUTH
+
+time_t dirvote_act(const or_options_t *options, time_t now);
+void dirvote_free_all(void);
+
+void dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens);
+void dirvote_clear_commits(networkstatus_t *ns);
+void dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
+                                    smartlist_t *dir_items);
+
+/* Storing signatures and votes functions */
+struct pending_vote_t * dirvote_add_vote(const char *vote_body,
+                                         const char **msg_out,
+                                         int *status_out);
+int dirvote_add_signatures(const char *detached_signatures_body,
+                           const char *source,
+                           const char **msg_out);
+
+#else /* HAVE_MODULE_DIRAUTH */
+
+static inline time_t
+dirvote_act(const or_options_t *options, time_t now)
+{
+  (void) options;
+  (void) now;
+  return TIME_MAX;
+}
+
+static inline void
+dirvote_free_all(void)
+{
+}
+
+static inline void
+dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens)
+{
+  (void) ns;
+  (void) tokens;
+}
+
+static inline void
+dirvote_clear_commits(networkstatus_t *ns)
+{
+  (void) ns;
+}
+
+static inline void
+dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
+                               smartlist_t *dir_items)
+{
+  (void) url;
+  (void) items;
+  (void) dir_items;
+}
+
+static inline struct pending_vote_t *
+dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
+{
+  (void) vote_body;
+  /* If the dirauth module is disabled, this should NEVER be called else we
+   * failed to safeguard the dirauth module. */
+  tor_assert_nonfatal_unreached();
+
+  /* We need to send out an error code. */
+  *status_out = 400;
+  *msg_out = "No directory authority support";
+  return NULL;
+}
+
+static inline int
+dirvote_add_signatures(const char *detached_signatures_body,
+                       const char *source,
+                       const char **msg_out)
+{
+  (void) detached_signatures_body;
+  (void) source;
+  (void) msg_out;
+  /* If the dirauth module is disabled, this should NEVER be called else we
+   * failed to safeguard the dirauth module. */
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+#endif /* HAVE_MODULE_DIRAUTH */
+
+/* Item access */
+MOCK_DECL(const char*, dirvote_get_pending_consensus,
+          (consensus_flavor_t flav));
+MOCK_DECL(const char*, dirvote_get_pending_detached_signatures, (void));
+const cached_dir_t *dirvote_get_vote(const char *fp, int flags);
+
+/*
+ * API used _only_ by the dirauth subsystem.
+ */
+
+void set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                      node_t *node,
+                                      routerinfo_t *ri, time_t now,
+                                      int listbadexits);
+networkstatus_t *
+dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
+                                        authority_cert_t *cert);
+
+vote_microdesc_hash_t *dirvote_format_all_microdesc_vote_lines(
+                                        const routerinfo_t *ri,
+                                        time_t now,
+                                        smartlist_t *microdescriptors_out);
+
+/*
+ * Exposed functions for unit tests.
+ */
+#ifdef DIRVOTE_PRIVATE
+
+/* Cert manipulation */
+STATIC authority_cert_t *authority_cert_dup(authority_cert_t *cert);
+STATIC int32_t dirvote_get_intermediate_param_value(
+                                   const smartlist_t *param_list,
+                                   const char *keyword,
+                                   int32_t default_val);
+STATIC char *format_networkstatus_vote(crypto_pk_t *private_key,
+                                 networkstatus_t *v3_ns);
+STATIC smartlist_t *dirvote_compute_params(smartlist_t *votes, int method,
+                             int total_authorities);
+STATIC char *compute_consensus_package_lines(smartlist_t *votes);
+STATIC char *make_consensus_method_list(int low, int high, const char *sep);
+STATIC int
+networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
+                                     int64_t M, int64_t E, int64_t D,
+                                     int64_t T, int64_t weight_scale);
+STATIC
+char *networkstatus_compute_consensus(smartlist_t *votes,
+                                      int total_authorities,
+                                      crypto_pk_t *identity_key,
+                                      crypto_pk_t *signing_key,
+                                      const char *legacy_identity_key_digest,
+                                      crypto_pk_t *legacy_signing_key,
+                                      consensus_flavor_t flavor);
+STATIC
+int networkstatus_add_detached_signatures(networkstatus_t *target,
+                                          ns_detached_signatures_t *sigs,
+                                          const char *source,
+                                          int severity,
+                                          const char **msg_out);
+STATIC
+char *networkstatus_get_detached_signatures(smartlist_t *consensuses);
+STATIC microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri,
+                                                   int consensus_method);
+
+#endif /* defined(DIRVOTE_PRIVATE) */
+
+#endif /* !defined(TOR_DIRVOTE_H) */
diff --git a/src/feature/dirauth/keypin.c b/src/feature/dirauth/keypin.c
new file mode 100644
index 0000000000..a2f3654b82
--- /dev/null
+++ b/src/feature/dirauth/keypin.c
@@ -0,0 +1,515 @@
+/* Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file keypin.c
+ *
+ * \brief Functions and structures for associating routers' RSA key
+ * fingerprints with their ED25519 keys.
+ */
+
+#define KEYPIN_PRIVATE
+
+#include "orconfig.h"
+
+#include "lib/cc/torint.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/binascii.h"
+#include "lib/encoding/time_fmt.h"
+#include "lib/fdio/fdio.h"
+#include "lib/fs/files.h"
+#include "lib/fs/mmap.h"
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/wallclock/approx_time.h"
+
+#include "ht.h"
+#include "or/keypin.h"
+
+#include "siphash.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#ifdef _WIN32
+#include <io.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+
+/**
+ * @file keypin.c
+ * @brief Key-pinning for RSA and Ed25519 identity keys at directory
+ *  authorities.
+ *
+ * Many older clients, and many internal interfaces, still refer to relays by
+ * their RSA1024 identity keys.  We can make this more secure, however:
+ * authorities use this module to track which RSA keys have been used along
+ * with which Ed25519 keys, and force such associations to be permanent.
+ *
+ * This module implements a key-pinning mechanism to ensure that it's safe
+ * to use RSA keys as identitifers even as we migrate to Ed25519 keys.  It
+ * remembers, for every Ed25519 key we've seen, what the associated Ed25519
+ * key is.  This way, if we see a different Ed25519 key with that RSA key,
+ * we'll know that there's a mismatch.
+ *
+ * (As of this writing, these key associations are advisory only, mostly
+ * because some relay operators kept mishandling their Ed25519 keys during
+ * the initial Ed25519 rollout.  We should fix this problem, and then toggle
+ * the AuthDirPinKeys option.)
+ *
+ * We persist these entries to disk using a simple format, where each line
+ * has a base64-encoded RSA SHA1 hash, then a base64-endoded Ed25519 key.
+ * Empty lines, misformed lines, and lines beginning with # are
+ * ignored. Lines beginning with @ are reserved for future extensions.
+ *
+ * The dirserv.c module is the main user of these functions.
+ */
+
+static int keypin_journal_append_entry(const uint8_t *rsa_id_digest,
+                                       const uint8_t *ed25519_id_key);
+static int keypin_check_and_add_impl(const uint8_t *rsa_id_digest,
+                                     const uint8_t *ed25519_id_key,
+                                     const int do_not_add,
+                                     const int replace);
+static int keypin_add_or_replace_entry_in_map(keypin_ent_t *ent);
+
+static HT_HEAD(rsamap, keypin_ent_st) the_rsa_map = HT_INITIALIZER();
+static HT_HEAD(edmap, keypin_ent_st) the_ed_map = HT_INITIALIZER();
+
+/** Hashtable helper: compare two keypin table entries and return true iff
+ * they have the same RSA key IDs. */
+static inline int
+keypin_ents_eq_rsa(const keypin_ent_t *a, const keypin_ent_t *b)
+{
+  return tor_memeq(a->rsa_id, b->rsa_id, sizeof(a->rsa_id));
+}
+
+/** Hashtable helper: hash a keypin table entries based on its RSA key ID */
+static inline unsigned
+keypin_ent_hash_rsa(const keypin_ent_t *a)
+{
+return (unsigned) siphash24g(a->rsa_id, sizeof(a->rsa_id));
+}
+
+/** Hashtable helper: compare two keypin table entries and return true iff
+ * they have the same ed25519 keys */
+static inline int
+keypin_ents_eq_ed(const keypin_ent_t *a, const keypin_ent_t *b)
+{
+  return tor_memeq(a->ed25519_key, b->ed25519_key, sizeof(a->ed25519_key));
+}
+
+/** Hashtable helper: hash a keypin table entries based on its ed25519 key */
+static inline unsigned
+keypin_ent_hash_ed(const keypin_ent_t *a)
+{
+return (unsigned) siphash24g(a->ed25519_key, sizeof(a->ed25519_key));
+}
+
+HT_PROTOTYPE(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa,
+               keypin_ents_eq_rsa)
+HT_GENERATE2(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa,
+               keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_)
+
+HT_PROTOTYPE(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed,
+               keypin_ents_eq_ed)
+HT_GENERATE2(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed,
+               keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_)
+
+/**
+ * Check whether we already have an entry in the key pinning table for a
+ * router with RSA ID digest <b>rsa_id_digest</b> or for ed25519 key
+ * <b>ed25519_id_key</b>.  If we have an entry that matches both keys,
+ * return KEYPIN_FOUND. If we find an entry that matches one key but
+ * not the other, return KEYPIN_MISMATCH.  If we have no entry for either
+ * key, add such an entry to the table and return KEYPIN_ADDED.
+ *
+ * If <b>replace_existing_entry</b> is true, then any time we would have said
+ * KEYPIN_FOUND, we instead add this entry anyway and return KEYPIN_ADDED.
+ */
+int
+keypin_check_and_add(const uint8_t *rsa_id_digest,
+                     const uint8_t *ed25519_id_key,
+                     const int replace_existing_entry)
+{
+  return keypin_check_and_add_impl(rsa_id_digest, ed25519_id_key, 0,
+                                   replace_existing_entry);
+}
+
+/**
+ * As keypin_check_and_add, but do not add.  Return KEYPIN_NOT_FOUND if
+ * we would add.
+ */
+int
+keypin_check(const uint8_t *rsa_id_digest,
+             const uint8_t *ed25519_id_key)
+{
+  return keypin_check_and_add_impl(rsa_id_digest, ed25519_id_key, 1, 0);
+}
+
+/**
+ * Helper: implements keypin_check and keypin_check_and_add.
+ */
+static int
+keypin_check_and_add_impl(const uint8_t *rsa_id_digest,
+                          const uint8_t *ed25519_id_key,
+                          const int do_not_add,
+                          const int replace)
+{
+  keypin_ent_t search, *ent;
+  memset(&search, 0, sizeof(search));
+  memcpy(search.rsa_id, rsa_id_digest, sizeof(search.rsa_id));
+  memcpy(search.ed25519_key, ed25519_id_key, sizeof(search.ed25519_key));
+
+  /* Search by RSA key digest first */
+  ent = HT_FIND(rsamap, &the_rsa_map, &search);
+  if (ent) {
+    tor_assert(fast_memeq(ent->rsa_id, rsa_id_digest, sizeof(ent->rsa_id)));
+    if (tor_memeq(ent->ed25519_key, ed25519_id_key,sizeof(ent->ed25519_key))) {
+      return KEYPIN_FOUND; /* Match on both keys. Great. */
+    } else {
+      if (!replace)
+        return KEYPIN_MISMATCH; /* Found RSA with different Ed key */
+    }
+  }
+
+  /* See if we know a different RSA key for this ed key */
+  if (! replace) {
+    ent = HT_FIND(edmap, &the_ed_map, &search);
+    if (ent) {
+      /* If we got here, then the ed key matches and the RSA doesn't */
+      tor_assert(fast_memeq(ent->ed25519_key, ed25519_id_key,
+                            sizeof(ent->ed25519_key)));
+      tor_assert(fast_memneq(ent->rsa_id, rsa_id_digest, sizeof(ent->rsa_id)));
+      return KEYPIN_MISMATCH;
+    }
+  }
+
+  /* Okay, this one is new to us. */
+  if (do_not_add)
+    return KEYPIN_NOT_FOUND;
+
+  ent = tor_memdup(&search, sizeof(search));
+  int r = keypin_add_or_replace_entry_in_map(ent);
+  if (! replace) {
+    tor_assert(r == 1);
+  } else {
+    tor_assert(r != 0);
+  }
+  keypin_journal_append_entry(rsa_id_digest, ed25519_id_key);
+  return KEYPIN_ADDED;
+}
+
+/**
+ * Helper: add <b>ent</b> to the hash tables.
+ */
+MOCK_IMPL(STATIC void,
+keypin_add_entry_to_map, (keypin_ent_t *ent))
+{
+  HT_INSERT(rsamap, &the_rsa_map, ent);
+  HT_INSERT(edmap, &the_ed_map, ent);
+}
+
+/**
+ * Helper: add 'ent' to the maps, replacing any entries that contradict it.
+ * Take ownership of 'ent', freeing it if needed.
+ *
+ * Return 0 if the entry was a duplicate, -1 if there was a conflict,
+ * and 1 if there was no conflict.
+ */
+static int
+keypin_add_or_replace_entry_in_map(keypin_ent_t *ent)
+{
+  int r = 1;
+  keypin_ent_t *ent2 = HT_FIND(rsamap, &the_rsa_map, ent);
+  keypin_ent_t *ent3 = HT_FIND(edmap, &the_ed_map, ent);
+  if (ent2 &&
+      fast_memeq(ent2->ed25519_key, ent->ed25519_key, DIGEST256_LEN)) {
+    /* We already have this mapping stored. Ignore it. */
+    tor_free(ent);
+    return 0;
+  } else if (ent2 || ent3) {
+    /* We have a conflict. (If we had no entry, we would have ent2 == ent3
+     * == NULL. If we had a non-conflicting duplicate, we would have found
+     * it above.)
+     *
+     * We respond by having this entry (ent) supersede all entries that it
+     * contradicts (ent2 and/or ent3). In other words, if we receive
+     * <rsa,ed>, we remove all <rsa,ed'> and all <rsa',ed>, for rsa'!=rsa
+     * and ed'!= ed.
+     */
+    const keypin_ent_t *t;
+    if (ent2) {
+      t = HT_REMOVE(rsamap, &the_rsa_map, ent2);
+      tor_assert(ent2 == t);
+      t = HT_REMOVE(edmap, &the_ed_map, ent2);
+      tor_assert(ent2 == t);
+    }
+    if (ent3 && ent2 != ent3) {
+      t = HT_REMOVE(rsamap, &the_rsa_map, ent3);
+      tor_assert(ent3 == t);
+      t = HT_REMOVE(edmap, &the_ed_map, ent3);
+      tor_assert(ent3 == t);
+      tor_free(ent3);
+    }
+    tor_free(ent2);
+    r = -1;
+    /* Fall through */
+  }
+
+  keypin_add_entry_to_map(ent);
+  return r;
+}
+
+/**
+ * Check whether we already have an entry in the key pinning table for a
+ * router with RSA ID digest <b>rsa_id_digest</b>.  If we have no such entry,
+ * return KEYPIN_NOT_FOUND.  If we find an entry that matches the RSA key but
+ * which has an ed25519 key, return KEYPIN_MISMATCH.
+ */
+int
+keypin_check_lone_rsa(const uint8_t *rsa_id_digest)
+{
+  keypin_ent_t search, *ent;
+  memset(&search, 0, sizeof(search));
+  memcpy(search.rsa_id, rsa_id_digest, sizeof(search.rsa_id));
+
+  /* Search by RSA key digest first */
+  ent = HT_FIND(rsamap, &the_rsa_map, &search);
+  if (ent) {
+    return KEYPIN_MISMATCH;
+  } else {
+    return KEYPIN_NOT_FOUND;
+  }
+}
+
+/** Open fd to the keypinning journal file. */
+static int keypin_journal_fd = -1;
+
+/** Open the key-pinning journal to append to <b>fname</b>.  Return 0 on
+ * success, -1 on failure. */
+int
+keypin_open_journal(const char *fname)
+{
+#ifndef O_SYNC
+#define O_SYNC 0
+#endif
+  int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY|O_SYNC, 0600);
+  if (fd < 0)
+    goto err;
+
+  if (tor_fd_seekend(fd) < 0)
+    goto err;
+
+  /* Add a newline in case the last line was only partially written */
+  if (write(fd, "\n", 1) < 1)
+    goto err;
+
+  /* Add something about when we opened this file. */
+  char buf[80];
+  char tbuf[ISO_TIME_LEN+1];
+  format_iso_time(tbuf, approx_time());
+  tor_snprintf(buf, sizeof(buf), "@opened-at %s\n", tbuf);
+  if (write_all_to_fd(fd, buf, strlen(buf)) < 0)
+    goto err;
+
+  keypin_journal_fd = fd;
+  return 0;
+ err:
+  if (fd >= 0)
+    close(fd);
+  return -1;
+}
+
+/** Close the keypinning journal file. */
+int
+keypin_close_journal(void)
+{
+  if (keypin_journal_fd >= 0)
+    close(keypin_journal_fd);
+  keypin_journal_fd = -1;
+  return 0;
+}
+
+/** Length of a keypinning journal line, including terminating newline. */
+#define JOURNAL_LINE_LEN (BASE64_DIGEST_LEN + BASE64_DIGEST256_LEN + 2)
+
+/** Add an entry to the keypinning journal to map <b>rsa_id_digest</b> and
+ * <b>ed25519_id_key</b>. */
+static int
+keypin_journal_append_entry(const uint8_t *rsa_id_digest,
+                            const uint8_t *ed25519_id_key)
+{
+  if (keypin_journal_fd == -1)
+    return -1;
+  char line[JOURNAL_LINE_LEN];
+  digest_to_base64(line, (const char*)rsa_id_digest);
+  line[BASE64_DIGEST_LEN] = ' ';
+  digest256_to_base64(line + BASE64_DIGEST_LEN + 1,
+                      (const char*)ed25519_id_key);
+  line[BASE64_DIGEST_LEN+1+BASE64_DIGEST256_LEN] = '\n';
+
+  if (write_all_to_fd(keypin_journal_fd, line, JOURNAL_LINE_LEN)<0) {
+    log_warn(LD_DIRSERV, "Error while adding a line to the key-pinning "
+             "journal: %s", strerror(errno));
+    keypin_close_journal();
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Load a journal from the <b>size</b>-byte region at <b>data</b>.  Return 0
+ * on success, -1 on failure. */
+STATIC int
+keypin_load_journal_impl(const char *data, size_t size)
+{
+  const char *start = data, *end = data + size, *next;
+
+  int n_corrupt_lines = 0;
+  int n_entries = 0;
+  int n_duplicates = 0;
+  int n_conflicts = 0;
+
+  for (const char *cp = start; cp < end; cp = next) {
+    const char *eol = memchr(cp, '\n', end-cp);
+    const char *eos = eol ? eol : end;
+    const size_t len = eos - cp;
+
+    next = eol ? eol + 1 : end;
+
+    if (len == 0) {
+      continue;
+    }
+
+    if (*cp == '@') {
+      /* Lines that start with @ are reserved. Ignore for now. */
+      continue;
+    }
+    if (*cp == '#') {
+      /* Lines that start with # are comments. */
+      continue;
+    }
+
+    /* Is it the right length?  (The -1 here is for the newline.) */
+    if (len != JOURNAL_LINE_LEN - 1) {
+      /* Lines with a bad length are corrupt unless they are empty.
+       * Ignore them either way */
+      for (const char *s = cp; s < eos; ++s) {
+        if (! TOR_ISSPACE(*s)) {
+          ++n_corrupt_lines;
+          break;
+        }
+      }
+      continue;
+    }
+
+    keypin_ent_t *ent = keypin_parse_journal_line(cp);
+
+    if (ent == NULL) {
+      ++n_corrupt_lines;
+      continue;
+    }
+
+    const int r = keypin_add_or_replace_entry_in_map(ent);
+    if (r == 0) {
+      ++n_duplicates;
+    } else if (r == -1) {
+      ++n_conflicts;
+    }
+
+    ++n_entries;
+  }
+
+  int severity = (n_corrupt_lines || n_duplicates) ? LOG_NOTICE : LOG_INFO;
+  tor_log(severity, LD_DIRSERV,
+          "Loaded %d entries from keypin journal. "
+          "Found %d corrupt lines (ignored), %d duplicates (harmless), "
+          "and %d conflicts (resolved in favor or more recent entry).",
+          n_entries, n_corrupt_lines, n_duplicates, n_conflicts);
+
+  return 0;
+}
+
+/**
+ * Load a journal from the file called <b>fname</b>. Return 0 on success,
+ * -1 on failure.
+ */
+int
+keypin_load_journal(const char *fname)
+{
+  tor_mmap_t *map = tor_mmap_file(fname);
+  if (!map) {
+    if (errno == ENOENT)
+      return 0;
+    else
+      return -1;
+  }
+  int r = keypin_load_journal_impl(map->data, map->size);
+  tor_munmap_file(map);
+  return r;
+}
+
+/** Parse a single keypinning journal line entry from <b>cp</b>.  The input
+ * does not need to be NUL-terminated, but it <em>does</em> need to have
+ * KEYPIN_JOURNAL_LINE_LEN -1 bytes available to read.  Return a new entry
+ * on success, and NULL on failure.
+ */
+STATIC keypin_ent_t *
+keypin_parse_journal_line(const char *cp)
+{
+  /* XXXX assumes !USE_OPENSSL_BASE64 */
+  keypin_ent_t *ent = tor_malloc_zero(sizeof(keypin_ent_t));
+
+  if (base64_decode((char*)ent->rsa_id, sizeof(ent->rsa_id),
+             cp, BASE64_DIGEST_LEN) != DIGEST_LEN ||
+      cp[BASE64_DIGEST_LEN] != ' ' ||
+      base64_decode((char*)ent->ed25519_key, sizeof(ent->ed25519_key),
+             cp+BASE64_DIGEST_LEN+1, BASE64_DIGEST256_LEN) != DIGEST256_LEN) {
+    tor_free(ent);
+    return NULL;
+  } else {
+    return ent;
+  }
+}
+
+/** Remove all entries from the keypinning table.*/
+void
+keypin_clear(void)
+{
+  int bad_entries = 0;
+  {
+    keypin_ent_t **ent, **next, *this;
+    for (ent = HT_START(rsamap, &the_rsa_map); ent != NULL; ent = next) {
+      this = *ent;
+      next = HT_NEXT_RMV(rsamap, &the_rsa_map, ent);
+
+      keypin_ent_t *other_ent = HT_REMOVE(edmap, &the_ed_map, this);
+      bad_entries += (other_ent != this);
+
+      tor_free(this);
+    }
+  }
+  bad_entries += HT_SIZE(&the_ed_map);
+
+  HT_CLEAR(edmap,&the_ed_map);
+  HT_CLEAR(rsamap,&the_rsa_map);
+
+  if (bad_entries) {
+    log_warn(LD_BUG, "Found %d discrepencies in the keypin database.",
+             bad_entries);
+  }
+}
diff --git a/src/feature/dirauth/keypin.h b/src/feature/dirauth/keypin.h
new file mode 100644
index 0000000000..73a76be563
--- /dev/null
+++ b/src/feature/dirauth/keypin.h
@@ -0,0 +1,47 @@
+/* Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_KEYPIN_H
+#define TOR_KEYPIN_H
+
+#include "lib/testsupport/testsupport.h"
+
+int keypin_check_and_add(const uint8_t *rsa_id_digest,
+                         const uint8_t *ed25519_id_key,
+                         const int replace_existing_entry);
+int keypin_check(const uint8_t *rsa_id_digest,
+                 const uint8_t *ed25519_id_key);
+
+int keypin_open_journal(const char *fname);
+int keypin_close_journal(void);
+int keypin_load_journal(const char *fname);
+void keypin_clear(void);
+int keypin_check_lone_rsa(const uint8_t *rsa_id_digest);
+
+#define KEYPIN_FOUND 0
+#define KEYPIN_ADDED 1
+#define KEYPIN_MISMATCH -1
+#define KEYPIN_NOT_FOUND -2
+
+#ifdef KEYPIN_PRIVATE
+
+/**
+ * In-memory representation of a key-pinning table entry.
+ */
+typedef struct keypin_ent_st {
+  HT_ENTRY(keypin_ent_st) rsamap_node;
+  HT_ENTRY(keypin_ent_st) edmap_node;
+  /** SHA1 hash of the RSA key */
+  uint8_t rsa_id[DIGEST_LEN];
+  /** Ed2219 key. */
+  uint8_t ed25519_key[DIGEST256_LEN];
+} keypin_ent_t;
+
+STATIC keypin_ent_t * keypin_parse_journal_line(const char *cp);
+STATIC int keypin_load_journal_impl(const char *data, size_t size);
+
+MOCK_DECL(STATIC void, keypin_add_entry_to_map, (keypin_ent_t *ent));
+#endif /* defined(KEYPIN_PRIVATE) */
+
+#endif /* !defined(TOR_KEYPIN_H) */
+
diff --git a/src/feature/dirauth/mode.h b/src/feature/dirauth/mode.h
new file mode 100644
index 0000000000..17c35aff64
--- /dev/null
+++ b/src/feature/dirauth/mode.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2018-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file mode.h
+ * \brief Standalone header file for directory authority mode.
+ **/
+
+#ifndef TOR_DIRAUTH_MODE_H
+#define TOR_DIRAUTH_MODE_H
+
+#ifdef HAVE_MODULE_DIRAUTH
+
+#include "or/router.h"
+
+/* Return true iff we believe ourselves to be a v3 authoritative directory
+ * server. */
+static inline int
+authdir_mode_v3(const or_options_t *options)
+{
+  return authdir_mode(options) && options->V3AuthoritativeDir != 0;
+}
+
+#else /* HAVE_MODULE_DIRAUTH */
+
+/* Without the dirauth module, we can't be a v3 directory authority, ever. */
+
+static inline int
+authdir_mode_v3(const or_options_t *options)
+{
+  (void) options;
+  return 0;
+}
+
+#endif /* HAVE_MODULE_DIRAUTH */
+
+#endif /* TOR_MODE_H */
+
diff --git a/src/feature/dirauth/ns_detached_signatures_st.h b/src/feature/dirauth/ns_detached_signatures_st.h
new file mode 100644
index 0000000000..26ceec84b9
--- /dev/null
+++ b/src/feature/dirauth/ns_detached_signatures_st.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NS_DETACHED_SIGNATURES_ST_H
+#define NS_DETACHED_SIGNATURES_ST_H
+
+/** A set of signatures for a networkstatus consensus.  Unless otherwise
+ * noted, all fields are as for networkstatus_t. */
+struct ns_detached_signatures_t {
+  time_t valid_after;
+  time_t fresh_until;
+  time_t valid_until;
+  strmap_t *digests; /**< Map from flavor name to digestset_t */
+  strmap_t *signatures; /**< Map from flavor name to list of
+                         * document_signature_t */
+};
+
+#endif
+
diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c
new file mode 100644
index 0000000000..d122690129
--- /dev/null
+++ b/src/feature/dirauth/shared_random.c
@@ -0,0 +1,1291 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shared_random.c
+ *
+ * \brief Functions and data structure needed to accomplish the shared
+ *        random protocol as defined in proposal #250.
+ *
+ * \details
+ *
+ * This file implements the dirauth-only commit-and-reveal protocol specified
+ * by proposal #250. The protocol has two phases (sr_phase_t): the commitment
+ * phase and the reveal phase (see get_sr_protocol_phase()).
+ *
+ * During the protocol, directory authorities keep state in memory (using
+ * sr_state_t) and in disk (using sr_disk_state_t). The synchronization between
+ * these two data structures happens in disk_state_update() and
+ * disk_state_parse().
+ *
+ * Here is a rough protocol outline:
+ *
+ *      1) In the beginning of the commitment phase, dirauths generate a
+ *         commitment/reveal value for the current protocol run (see
+ *         new_protocol_run() and sr_generate_our_commit()).
+ *
+ *      2) During voting, dirauths publish their commits in their votes
+ *         depending on the current phase.  Dirauths also include the two
+ *         latest shared random values (SRV) in their votes.
+ *         (see sr_get_string_for_vote())
+ *
+ *      3) Upon receiving a commit from a vote, authorities parse it, verify
+ *         it, and attempt to save any new commitment or reveal information in
+ *         their state file (see extract_shared_random_commits() and
+ *         sr_handle_received_commits()).  They also parse SRVs from votes to
+ *         decide which SRV should be included in the final consensus (see
+ *         extract_shared_random_srvs()).
+ *
+ *      3) After voting is done, we count the SRVs we extracted from the votes,
+ *         to find the one voted by the majority of dirauths which should be
+ *         included in the final consensus (see get_majority_srv_from_votes()).
+ *         If an appropriate SRV is found, it is embedded in the consensus (see
+ *         sr_get_string_for_consensus()).
+ *
+ *      4) At the end of the reveal phase, dirauths compute a fresh SRV for the
+ *         day using the active commits (see sr_compute_srv()).  This new SRV
+ *         is embedded in the votes as described above.
+ *
+ * Some more notes:
+ *
+ * - To support rebooting authorities and to avoid double voting, each dirauth
+ *   saves the current state of the protocol on disk so that it can resume
+ *   normally in case of reboot. The disk state (sr_disk_state_t) is managed by
+ *   shared_random_state.c:state_query() and we go to extra lengths to ensure
+ *   that the state is flushed on disk everytime we receive any useful
+ *   information like commits or SRVs.
+ *
+ * - When we receive a commit from a vote, we examine it to see if it's useful
+ *   to us and whether it's appropriate to receive it according to the current
+ *   phase of the protocol (see should_keep_commit()). If the commit is useful
+ *   to us, we save it in our disk state using save_commit_to_state().  When we
+ *   receive the reveal information corresponding to a commitment, we verify
+ *   that they indeed match using verify_commit_and_reveal().
+ *
+ * - We treat consensuses as the ground truth, so everytime we generate a new
+ *   consensus we update our SR state accordingly even if our local view was
+ *   different (see sr_act_post_consensus()).
+ *
+ * - After a consensus has been composed, the SR protocol state gets prepared
+ *   for the next voting session using sr_state_update(). That function takes
+ *   care of housekeeping and also rotates the SRVs and commits in case a new
+ *   protocol run is coming up. We also call sr_state_update() on bootup (in
+ *   sr_state_init()), to prepare the state for the very first voting session.
+ *
+ * Terminology:
+ *
+ * - "Commitment" is the commitment value of the commit-and-reveal protocol.
+ *
+ * - "Reveal" is the reveal value of the commit-and-reveal protocol.
+ *
+ * - "Commit" is a struct (sr_commit_t) that contains a commitment value and
+ *    optionally also a corresponding reveal value.
+ *
+ * - "SRV" is the Shared Random Value that gets generated as the result of the
+ *   commit-and-reveal protocol.
+ **/
+
+#define SHARED_RANDOM_PRIVATE
+
+#include "or/or.h"
+#include "or/dirauth/shared_random.h"
+#include "or/config.h"
+#include "or/confparse.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/networkstatus.h"
+#include "or/router.h"
+#include "or/routerkeys.h"
+#include "or/routerlist.h"
+#include "or/shared_random_client.h"
+#include "or/dirauth/shared_random_state.h"
+#include "or/voting_schedule.h"
+
+#include "or/dirauth/dirvote.h"
+#include "or/dirauth/mode.h"
+
+#include "or/authority_cert_st.h"
+#include "or/networkstatus_st.h"
+
+/* String prefix of shared random values in votes/consensuses. */
+static const char previous_srv_str[] = "shared-rand-previous-value";
+static const char current_srv_str[] = "shared-rand-current-value";
+static const char commit_ns_str[] = "shared-rand-commit";
+static const char sr_flag_ns_str[] = "shared-rand-participate";
+
+/* The value of the consensus param AuthDirNumSRVAgreements found in the
+ * vote. This is set once the consensus creation subsystem requests the
+ * SRV(s) that should be put in the consensus. We use this value to decide
+ * if we keep or not an SRV. */
+static int32_t num_srv_agreements_from_vote;
+
+/* Return a heap allocated copy of the SRV <b>orig</b>. */
+STATIC sr_srv_t *
+srv_dup(const sr_srv_t *orig)
+{
+  sr_srv_t *duplicate = NULL;
+
+  if (!orig) {
+    return NULL;
+  }
+
+  duplicate = tor_malloc_zero(sizeof(sr_srv_t));
+  duplicate->num_reveals = orig->num_reveals;
+  memcpy(duplicate->value, orig->value, sizeof(duplicate->value));
+  return duplicate;
+}
+
+/* Allocate a new commit object and initializing it with <b>rsa_identity</b>
+ * that MUST be provided. The digest algorithm is set to the default one
+ * that is supported. The rest is uninitialized. This never returns NULL. */
+static sr_commit_t *
+commit_new(const char *rsa_identity)
+{
+  sr_commit_t *commit;
+
+  tor_assert(rsa_identity);
+
+  commit = tor_malloc_zero(sizeof(*commit));
+  commit->alg = SR_DIGEST_ALG;
+  memcpy(commit->rsa_identity, rsa_identity, sizeof(commit->rsa_identity));
+  base16_encode(commit->rsa_identity_hex, sizeof(commit->rsa_identity_hex),
+                commit->rsa_identity, sizeof(commit->rsa_identity));
+  return commit;
+}
+
+/* Issue a log message describing <b>commit</b>. */
+static void
+commit_log(const sr_commit_t *commit)
+{
+  tor_assert(commit);
+
+  log_debug(LD_DIR, "SR: Commit from %s", sr_commit_get_rsa_fpr(commit));
+  log_debug(LD_DIR, "SR: Commit: [TS: %" PRIu64 "] [Encoded: %s]",
+            commit->commit_ts, commit->encoded_commit);
+  log_debug(LD_DIR, "SR: Reveal: [TS: %" PRIu64 "] [Encoded: %s]",
+            commit->reveal_ts, safe_str(commit->encoded_reveal));
+}
+
+/* Make sure that the commitment and reveal information in <b>commit</b>
+ * match. If they match return 0, return -1 otherwise. This function MUST be
+ * used everytime we receive a new reveal value. Furthermore, the commit
+ * object MUST have a reveal value and the hash of the reveal value. */
+STATIC int
+verify_commit_and_reveal(const sr_commit_t *commit)
+{
+  tor_assert(commit);
+
+  log_debug(LD_DIR, "SR: Validating commit from authority %s",
+            sr_commit_get_rsa_fpr(commit));
+
+  /* Check that the timestamps match. */
+  if (commit->commit_ts != commit->reveal_ts) {
+    log_warn(LD_BUG, "SR: Commit timestamp %" PRIu64 " doesn't match reveal "
+                     "timestamp %" PRIu64, commit->commit_ts,
+             commit->reveal_ts);
+    goto invalid;
+  }
+
+  /* Verify that the hashed_reveal received in the COMMIT message, matches
+   * the reveal we just received. */
+  {
+    /* We first hash the reveal we just received. */
+    char received_hashed_reveal[sizeof(commit->hashed_reveal)];
+
+    /* Only sha3-256 is supported. */
+    if (commit->alg != SR_DIGEST_ALG) {
+      goto invalid;
+    }
+
+    /* Use the invariant length since the encoded reveal variable has an
+     * extra byte for the NUL terminated byte. */
+    if (crypto_digest256(received_hashed_reveal, commit->encoded_reveal,
+                         SR_REVEAL_BASE64_LEN, commit->alg) < 0) {
+      /* Unable to digest the reveal blob, this is unlikely. */
+      goto invalid;
+    }
+
+    /* Now compare that with the hashed_reveal we received in COMMIT. */
+    if (fast_memneq(received_hashed_reveal, commit->hashed_reveal,
+                    sizeof(received_hashed_reveal))) {
+      log_warn(LD_BUG, "SR: Received reveal value from authority %s "
+                       "doesn't match the commit value.",
+               sr_commit_get_rsa_fpr(commit));
+      goto invalid;
+    }
+  }
+
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* Return true iff the commit contains an encoded reveal value. */
+STATIC int
+commit_has_reveal_value(const sr_commit_t *commit)
+{
+  return !tor_mem_is_zero(commit->encoded_reveal,
+                          sizeof(commit->encoded_reveal));
+}
+
+/* Parse the encoded commit. The format is:
+ *    base64-encode( TIMESTAMP || H(REVEAL) )
+ *
+ * If successfully decoded and parsed, commit is updated and 0 is returned.
+ * On error, return -1. */
+STATIC int
+commit_decode(const char *encoded, sr_commit_t *commit)
+{
+  int decoded_len = 0;
+  size_t offset = 0;
+  char b64_decoded[SR_COMMIT_LEN];
+
+  tor_assert(encoded);
+  tor_assert(commit);
+
+  if (strlen(encoded) > SR_COMMIT_BASE64_LEN) {
+    /* This means that if we base64 decode successfully the reveiced commit,
+     * we'll end up with a bigger decoded commit thus unusable. */
+    goto error;
+  }
+
+  /* Decode our encoded commit. Let's be careful here since _encoded_ is
+   * coming from the network in a dirauth vote so we expect nothing more
+   * than the base64 encoded length of a commit. */
+  decoded_len = base64_decode(b64_decoded, sizeof(b64_decoded),
+                              encoded, strlen(encoded));
+  if (decoded_len < 0) {
+    log_warn(LD_BUG, "SR: Commit from authority %s can't be decoded.",
+             sr_commit_get_rsa_fpr(commit));
+    goto error;
+  }
+
+  if (decoded_len != SR_COMMIT_LEN) {
+    log_warn(LD_BUG, "SR: Commit from authority %s decoded length doesn't "
+                     "match the expected length (%d vs %u).",
+             sr_commit_get_rsa_fpr(commit), decoded_len,
+             (unsigned)SR_COMMIT_LEN);
+    goto error;
+  }
+
+  /* First is the timestamp (8 bytes). */
+  commit->commit_ts = tor_ntohll(get_uint64(b64_decoded));
+  offset += sizeof(uint64_t);
+  /* Next is hashed reveal. */
+  memcpy(commit->hashed_reveal, b64_decoded + offset,
+         sizeof(commit->hashed_reveal));
+  /* Copy the base64 blob to the commit. Useful for voting. */
+  strlcpy(commit->encoded_commit, encoded, sizeof(commit->encoded_commit));
+
+  return 0;
+
+ error:
+  return -1;
+}
+
+/* Parse the b64 blob at <b>encoded</b> containing reveal information and
+ * store the information in-place in <b>commit</b>. Return 0 on success else
+ * a negative value. */
+STATIC int
+reveal_decode(const char *encoded, sr_commit_t *commit)
+{
+  int decoded_len = 0;
+  char b64_decoded[SR_REVEAL_LEN];
+
+  tor_assert(encoded);
+  tor_assert(commit);
+
+  if (strlen(encoded) > SR_REVEAL_BASE64_LEN) {
+    /* This means that if we base64 decode successfully the received reveal
+     * value, we'll end up with a bigger decoded value thus unusable. */
+    goto error;
+  }
+
+  /* Decode our encoded reveal. Let's be careful here since _encoded_ is
+   * coming from the network in a dirauth vote so we expect nothing more
+   * than the base64 encoded length of our reveal. */
+  decoded_len = base64_decode(b64_decoded, sizeof(b64_decoded),
+                              encoded, strlen(encoded));
+  if (decoded_len < 0) {
+    log_warn(LD_BUG, "SR: Reveal from authority %s can't be decoded.",
+             sr_commit_get_rsa_fpr(commit));
+    goto error;
+  }
+
+  if (decoded_len != SR_REVEAL_LEN) {
+    log_warn(LD_BUG, "SR: Reveal from authority %s decoded length is "
+                     "doesn't match the expected length (%d vs %u)",
+             sr_commit_get_rsa_fpr(commit), decoded_len,
+             (unsigned)SR_REVEAL_LEN);
+    goto error;
+  }
+
+  commit->reveal_ts = tor_ntohll(get_uint64(b64_decoded));
+  /* Copy the last part, the random value. */
+  memcpy(commit->random_number, b64_decoded + 8,
+         sizeof(commit->random_number));
+  /* Also copy the whole message to use during verification */
+  strlcpy(commit->encoded_reveal, encoded, sizeof(commit->encoded_reveal));
+
+  return 0;
+
+ error:
+  return -1;
+}
+
+/* Encode a reveal element using a given commit object to dst which is a
+ * buffer large enough to put the base64-encoded reveal construction. The
+ * format is as follow:
+ *     REVEAL = base64-encode( TIMESTAMP || H(RN) )
+ * Return base64 encoded length on success else a negative value.
+ */
+STATIC int
+reveal_encode(const sr_commit_t *commit, char *dst, size_t len)
+{
+  int ret;
+  size_t offset = 0;
+  char buf[SR_REVEAL_LEN] = {0};
+
+  tor_assert(commit);
+  tor_assert(dst);
+
+  set_uint64(buf, tor_htonll(commit->reveal_ts));
+  offset += sizeof(uint64_t);
+  memcpy(buf + offset, commit->random_number,
+         sizeof(commit->random_number));
+
+  /* Let's clean the buffer and then b64 encode it. */
+  memset(dst, 0, len);
+  ret = base64_encode(dst, len, buf, sizeof(buf), 0);
+  /* Wipe this buffer because it contains our random value. */
+  memwipe(buf, 0, sizeof(buf));
+  return ret;
+}
+
+/* Encode the given commit object to dst which is a buffer large enough to
+ * put the base64-encoded commit. The format is as follow:
+ *     COMMIT = base64-encode( TIMESTAMP || H(H(RN)) )
+ * Return base64 encoded length on success else a negative value.
+ */
+STATIC int
+commit_encode(const sr_commit_t *commit, char *dst, size_t len)
+{
+  size_t offset = 0;
+  char buf[SR_COMMIT_LEN] = {0};
+
+  tor_assert(commit);
+  tor_assert(dst);
+
+  /* First is the timestamp (8 bytes). */
+  set_uint64(buf, tor_htonll(commit->commit_ts));
+  offset += sizeof(uint64_t);
+  /* and then the hashed reveal. */
+  memcpy(buf + offset, commit->hashed_reveal,
+         sizeof(commit->hashed_reveal));
+
+  /* Clean the buffer and then b64 encode it. */
+  memset(dst, 0, len);
+  return base64_encode(dst, len, buf, sizeof(buf), 0);
+}
+
+/* Cleanup both our global state and disk state. */
+static void
+sr_cleanup(void)
+{
+  sr_state_free_all();
+}
+
+/* Using <b>commit</b>, return a newly allocated string containing the commit
+ * information that should be used during SRV calculation. It's the caller
+ * responsibility to free the memory. Return NULL if this is not a commit to be
+ * used for SRV calculation. */
+static char *
+get_srv_element_from_commit(const sr_commit_t *commit)
+{
+  char *element;
+  tor_assert(commit);
+
+  if (!commit_has_reveal_value(commit)) {
+    return NULL;
+  }
+
+  tor_asprintf(&element, "%s%s", sr_commit_get_rsa_fpr(commit),
+               commit->encoded_reveal);
+  return element;
+}
+
+/* Return a srv object that is built with the construction:
+ *    SRV = SHA3-256("shared-random" | INT_8(reveal_num) |
+ *                   INT_4(version) | HASHED_REVEALS | previous_SRV)
+ * This function cannot fail. */
+static sr_srv_t *
+generate_srv(const char *hashed_reveals, uint64_t reveal_num,
+             const sr_srv_t *previous_srv)
+{
+  char msg[DIGEST256_LEN + SR_SRV_MSG_LEN] = {0};
+  size_t offset = 0;
+  sr_srv_t *srv;
+
+  tor_assert(hashed_reveals);
+
+  /* Add the invariant token. */
+  memcpy(msg, SR_SRV_TOKEN, SR_SRV_TOKEN_LEN);
+  offset += SR_SRV_TOKEN_LEN;
+  set_uint64(msg + offset, tor_htonll(reveal_num));
+  offset += sizeof(uint64_t);
+  set_uint32(msg + offset, htonl(SR_PROTO_VERSION));
+  offset += sizeof(uint32_t);
+  memcpy(msg + offset, hashed_reveals, DIGEST256_LEN);
+  offset += DIGEST256_LEN;
+  if (previous_srv != NULL) {
+    memcpy(msg + offset, previous_srv->value, sizeof(previous_srv->value));
+  }
+
+  /* Ok we have our message and key for the HMAC computation, allocate our
+   * srv object and do the last step. */
+  srv = tor_malloc_zero(sizeof(*srv));
+  crypto_digest256((char *) srv->value, msg, sizeof(msg), SR_DIGEST_ALG);
+  srv->num_reveals = reveal_num;
+
+  {
+    /* Debugging. */
+    char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1];
+    sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv);
+    log_info(LD_DIR, "SR: Generated SRV: %s", srv_hash_encoded);
+  }
+  return srv;
+}
+
+/* Compare reveal values and return the result. This should exclusively be
+ * used by smartlist_sort(). */
+static int
+compare_reveal_(const void **_a, const void **_b)
+{
+  const sr_commit_t *a = *_a, *b = *_b;
+  return fast_memcmp(a->hashed_reveal, b->hashed_reveal,
+                     sizeof(a->hashed_reveal));
+}
+
+/* Given <b>commit</b> give the line that we should place in our votes.
+ * It's the responsibility of the caller to free the string. */
+static char *
+get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
+{
+  char *vote_line = NULL;
+
+  switch (phase) {
+  case SR_PHASE_COMMIT:
+    tor_asprintf(&vote_line, "%s %u %s %s %s\n",
+                 commit_ns_str,
+                 SR_PROTO_VERSION,
+                 crypto_digest_algorithm_get_name(commit->alg),
+                 sr_commit_get_rsa_fpr(commit),
+                 commit->encoded_commit);
+    break;
+  case SR_PHASE_REVEAL:
+  {
+    /* Send a reveal value for this commit if we have one. */
+    const char *reveal_str = commit->encoded_reveal;
+    if (tor_mem_is_zero(commit->encoded_reveal,
+                        sizeof(commit->encoded_reveal))) {
+      reveal_str = "";
+    }
+    tor_asprintf(&vote_line, "%s %u %s %s %s %s\n",
+                 commit_ns_str,
+                 SR_PROTO_VERSION,
+                 crypto_digest_algorithm_get_name(commit->alg),
+                 sr_commit_get_rsa_fpr(commit),
+                 commit->encoded_commit, reveal_str);
+    break;
+  }
+  default:
+    tor_assert(0);
+  }
+
+  log_debug(LD_DIR, "SR: Commit vote line: %s", vote_line);
+  return vote_line;
+}
+
+/* Return a heap allocated string that contains the given <b>srv</b> string
+ * representation formatted for a networkstatus document using the
+ * <b>key</b> as the start of the line. This doesn't return NULL. */
+static char *
+srv_to_ns_string(const sr_srv_t *srv, const char *key)
+{
+  char *srv_str;
+  char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1];
+  tor_assert(srv);
+  tor_assert(key);
+
+  sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv);
+  tor_asprintf(&srv_str, "%s %" PRIu64 " %s\n", key,
+               srv->num_reveals, srv_hash_encoded);
+  log_debug(LD_DIR, "SR: Consensus SRV line: %s", srv_str);
+  return srv_str;
+}
+
+/* Given the previous SRV and the current SRV, return a heap allocated
+ * string with their data that could be put in a vote or a consensus. Caller
+ * must free the returned string.  Return NULL if no SRVs were provided. */
+static char *
+get_ns_str_from_sr_values(const sr_srv_t *prev_srv, const sr_srv_t *cur_srv)
+{
+  smartlist_t *chunks = NULL;
+  char *srv_str;
+
+  if (!prev_srv && !cur_srv) {
+    return NULL;
+  }
+
+  chunks = smartlist_new();
+
+  if (prev_srv) {
+    char *srv_line = srv_to_ns_string(prev_srv, previous_srv_str);
+    smartlist_add(chunks, srv_line);
+  }
+
+  if (cur_srv) {
+    char *srv_line = srv_to_ns_string(cur_srv, current_srv_str);
+    smartlist_add(chunks, srv_line);
+  }
+
+  /* Join the line(s) here in one string to return. */
+  srv_str = smartlist_join_strings(chunks, "", 0, NULL);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+
+  return srv_str;
+}
+
+/* Return 1 iff the two commits have the same commitment values. This
+ * function does not care about reveal values. */
+STATIC int
+commitments_are_the_same(const sr_commit_t *commit_one,
+                         const sr_commit_t *commit_two)
+{
+  tor_assert(commit_one);
+  tor_assert(commit_two);
+
+  if (strcmp(commit_one->encoded_commit, commit_two->encoded_commit)) {
+    return 0;
+  }
+  return 1;
+}
+
+/* We just received a commit from the vote of authority with
+ * <b>identity_digest</b>. Return 1 if this commit is authorititative that
+ * is, it belongs to the authority that voted it. Else return 0 if not. */
+STATIC int
+commit_is_authoritative(const sr_commit_t *commit,
+                        const char *voter_key)
+{
+  tor_assert(commit);
+  tor_assert(voter_key);
+
+  return fast_memeq(commit->rsa_identity, voter_key,
+                    sizeof(commit->rsa_identity));
+}
+
+/* Decide if the newly received <b>commit</b> should be kept depending on
+ * the current phase and state of the protocol. The <b>voter_key</b> is the
+ * RSA identity key fingerprint of the authority's vote from which the
+ * commit comes from. The <b>phase</b> is the phase we should be validating
+ * the commit for. Return 1 if the commit should be added to our state or 0
+ * if not. */
+STATIC int
+should_keep_commit(const sr_commit_t *commit, const char *voter_key,
+                   sr_phase_t phase)
+{
+  const sr_commit_t *saved_commit;
+
+  tor_assert(commit);
+  tor_assert(voter_key);
+
+  log_debug(LD_DIR, "SR: Inspecting commit from %s (voter: %s)?",
+            sr_commit_get_rsa_fpr(commit),
+            hex_str(voter_key, DIGEST_LEN));
+
+  /* For a commit to be considered, it needs to be authoritative (it should
+   * be the voter's own commit). */
+  if (!commit_is_authoritative(commit, voter_key)) {
+    log_debug(LD_DIR, "SR: Ignoring non-authoritative commit.");
+    goto ignore;
+  }
+
+  /* Let's make sure, for extra safety, that this fingerprint is known to
+   * us. Even though this comes from a vote, doesn't hurt to be
+   * extracareful. */
+  if (trusteddirserver_get_by_v3_auth_digest(commit->rsa_identity) == NULL) {
+    log_warn(LD_DIR, "SR: Fingerprint %s is not from a recognized "
+                     "authority. Discarding commit.",
+             escaped(commit->rsa_identity));
+    goto ignore;
+  }
+
+  /* Check if the authority that voted for <b>commit</b> has already posted
+   * a commit before. */
+  saved_commit = sr_state_get_commit(commit->rsa_identity);
+
+  switch (phase) {
+  case SR_PHASE_COMMIT:
+    /* Already having a commit for an authority so ignore this one. */
+    if (saved_commit) {
+      /*  Receiving known commits should happen naturally since commit phase
+          lasts multiple rounds. However if the commitment value changes
+          during commit phase, it might be a bug so log more loudly. */
+      if (!commitments_are_the_same(commit, saved_commit)) {
+        log_info(LD_DIR,
+                 "SR: Received altered commit from %s in commit phase.",
+                 sr_commit_get_rsa_fpr(commit));
+      } else {
+        log_debug(LD_DIR, "SR: Ignoring known commit during commit phase.");
+      }
+      goto ignore;
+    }
+
+    /* A commit with a reveal value during commitment phase is very wrong. */
+    if (commit_has_reveal_value(commit)) {
+      log_warn(LD_DIR, "SR: Commit from authority %s has a reveal value "
+                       "during COMMIT phase. (voter: %s)",
+               sr_commit_get_rsa_fpr(commit),
+               hex_str(voter_key, DIGEST_LEN));
+      goto ignore;
+    }
+    break;
+  case SR_PHASE_REVEAL:
+    /* We are now in reveal phase. We keep a commit if and only if:
+     *
+     * - We have already seen a commit by this auth, AND
+     * - the saved commit has the same commitment value as this one, AND
+     * - the saved commit has no reveal information, AND
+     * - this commit does have reveal information, AND
+     * - the reveal & commit information are matching.
+     *
+     * If all the above are true, then we are interested in this new commit
+     * for its reveal information. */
+
+    if (!saved_commit) {
+      log_debug(LD_DIR, "SR: Ignoring commit first seen in reveal phase.");
+      goto ignore;
+    }
+
+    if (!commitments_are_the_same(commit, saved_commit)) {
+      log_warn(LD_DIR, "SR: Commit from authority %s is different from "
+                       "previous commit in our state (voter: %s)",
+               sr_commit_get_rsa_fpr(commit),
+               hex_str(voter_key, DIGEST_LEN));
+      goto ignore;
+    }
+
+    if (commit_has_reveal_value(saved_commit)) {
+      log_debug(LD_DIR, "SR: Ignoring commit with known reveal info.");
+      goto ignore;
+    }
+
+    if (!commit_has_reveal_value(commit)) {
+      log_debug(LD_DIR, "SR: Ignoring commit without reveal value.");
+      goto ignore;
+    }
+
+    if (verify_commit_and_reveal(commit) < 0) {
+      log_warn(LD_BUG, "SR: Commit from authority %s has an invalid "
+                       "reveal value. (voter: %s)",
+               sr_commit_get_rsa_fpr(commit),
+               hex_str(voter_key, DIGEST_LEN));
+      goto ignore;
+    }
+    break;
+  default:
+    tor_assert(0);
+  }
+
+  return 1;
+
+ ignore:
+  return 0;
+}
+
+/* We are in reveal phase and we found a valid and verified <b>commit</b> in
+ * a vote that contains reveal values that we could use. Update the commit
+ * we have in our state. Never call this with an unverified commit. */
+STATIC void
+save_commit_during_reveal_phase(const sr_commit_t *commit)
+{
+  sr_commit_t *saved_commit;
+
+  tor_assert(commit);
+
+  /* Get the commit from our state. */
+  saved_commit = sr_state_get_commit(commit->rsa_identity);
+  tor_assert(saved_commit);
+  /* Safety net. They can not be different commitments at this point. */
+  int same_commits = commitments_are_the_same(commit, saved_commit);
+  tor_assert(same_commits);
+
+  /* Copy reveal information to our saved commit. */
+  sr_state_copy_reveal_info(saved_commit, commit);
+}
+
+/* Save <b>commit</b> to our persistent state. Depending on the current
+ * phase, different actions are taken. Steals reference of <b>commit</b>.
+ * The commit object MUST be valid and verified before adding it to the
+ * state. */
+STATIC void
+save_commit_to_state(sr_commit_t *commit)
+{
+  sr_phase_t phase = sr_state_get_phase();
+
+  ASSERT_COMMIT_VALID(commit);
+
+  switch (phase) {
+  case SR_PHASE_COMMIT:
+    /* During commit phase, just save any new authoritative commit */
+    sr_state_add_commit(commit);
+    break;
+  case SR_PHASE_REVEAL:
+    save_commit_during_reveal_phase(commit);
+    sr_commit_free(commit);
+    break;
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Return 1 if we should we keep an SRV voted by <b>n_agreements</b> auths.
+ * Return 0 if we should ignore it. */
+static int
+should_keep_srv(int n_agreements)
+{
+  /* Check if the most popular SRV has reached majority. */
+  int n_voters = get_n_authorities(V3_DIRINFO);
+  int votes_required_for_majority = (n_voters / 2) + 1;
+
+  /* We need at the very least majority to keep a value. */
+  if (n_agreements < votes_required_for_majority) {
+    log_notice(LD_DIR, "SR: SRV didn't reach majority [%d/%d]!",
+               n_agreements, votes_required_for_majority);
+    return 0;
+  }
+
+  /* When we just computed a new SRV, we need to have super majority in order
+   * to keep it. */
+  if (sr_state_srv_is_fresh()) {
+    /* Check if we have super majority for this new SRV value. */
+    if (n_agreements < num_srv_agreements_from_vote) {
+      log_notice(LD_DIR, "SR: New SRV didn't reach agreement [%d/%d]!",
+                 n_agreements, num_srv_agreements_from_vote);
+      return 0;
+    }
+  }
+
+  return 1;
+}
+
+/* Helper: compare two DIGEST256_LEN digests. */
+static int
+compare_srvs_(const void **_a, const void **_b)
+{
+  const sr_srv_t *a = *_a, *b = *_b;
+  return tor_memcmp(a->value, b->value, sizeof(a->value));
+}
+
+/* Return the most frequent member of the sorted list of DIGEST256_LEN
+ * digests in <b>sl</b> with the count of that most frequent element. */
+static sr_srv_t *
+smartlist_get_most_frequent_srv(const smartlist_t *sl, int *count_out)
+{
+  return smartlist_get_most_frequent_(sl, compare_srvs_, count_out);
+}
+
+/** Compare two SRVs. Used in smartlist sorting. */
+static int
+compare_srv_(const void **_a, const void **_b)
+{
+  const sr_srv_t *a = *_a, *b = *_b;
+  return fast_memcmp(a->value, b->value,
+                     sizeof(a->value));
+}
+
+/* Using a list of <b>votes</b>, return the SRV object from them that has
+ * been voted by the majority of dirauths. If <b>current</b> is set, we look
+ * for the current SRV value else the previous one. The returned pointer is
+ * an object located inside a vote. NULL is returned if no appropriate value
+ * could be found. */
+STATIC sr_srv_t *
+get_majority_srv_from_votes(const smartlist_t *votes, int current)
+{
+  int count = 0;
+  sr_srv_t *most_frequent_srv = NULL;
+  sr_srv_t *the_srv = NULL;
+  smartlist_t *srv_list;
+
+  tor_assert(votes);
+
+  srv_list = smartlist_new();
+
+  /* Walk over votes and register any SRVs found. */
+  SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
+    sr_srv_t *srv_tmp = NULL;
+
+    if (!v->sr_info.participate) {
+      /* Ignore vote that do not participate. */
+      continue;
+    }
+    /* Do we want previous or current SRV? */
+    srv_tmp = current ? v->sr_info.current_srv : v->sr_info.previous_srv;
+    if (!srv_tmp) {
+      continue;
+    }
+
+    smartlist_add(srv_list, srv_tmp);
+  } SMARTLIST_FOREACH_END(v);
+
+  smartlist_sort(srv_list, compare_srv_);
+  most_frequent_srv = smartlist_get_most_frequent_srv(srv_list, &count);
+  if (!most_frequent_srv) {
+    goto end;
+  }
+
+  /* Was this SRV voted by enough auths for us to keep it? */
+  if (!should_keep_srv(count)) {
+    goto end;
+  }
+
+  /* We found an SRV that we can use! Habemus SRV! */
+  the_srv = most_frequent_srv;
+
+  {
+    /* Debugging */
+    char encoded[SR_SRV_VALUE_BASE64_LEN + 1];
+    sr_srv_encode(encoded, sizeof(encoded), the_srv);
+    log_debug(LD_DIR, "SR: Chosen SRV by majority: %s (%d votes)", encoded,
+              count);
+  }
+
+ end:
+  /* We do not free any sr_srv_t values, we don't have the ownership. */
+  smartlist_free(srv_list);
+  return the_srv;
+}
+
+/* Free a commit object. */
+void
+sr_commit_free_(sr_commit_t *commit)
+{
+  if (commit == NULL) {
+    return;
+  }
+  /* Make sure we do not leave OUR random number in memory. */
+  memwipe(commit->random_number, 0, sizeof(commit->random_number));
+  tor_free(commit);
+}
+
+/* Generate the commitment/reveal value for the protocol run starting at
+ * <b>timestamp</b>. <b>my_rsa_cert</b> is our authority RSA certificate. */
+sr_commit_t *
+sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert)
+{
+  sr_commit_t *commit = NULL;
+  char digest[DIGEST_LEN];
+
+  tor_assert(my_rsa_cert);
+
+  /* Get our RSA identity fingerprint */
+  if (crypto_pk_get_digest(my_rsa_cert->identity_key, digest) < 0) {
+    goto error;
+  }
+
+  /* New commit with our identity key. */
+  commit = commit_new(digest);
+
+  /* Generate the reveal random value */
+  crypto_strongest_rand(commit->random_number,
+                        sizeof(commit->random_number));
+  commit->commit_ts = commit->reveal_ts = timestamp;
+
+  /* Now get the base64 blob that corresponds to our reveal */
+  if (reveal_encode(commit, commit->encoded_reveal,
+                    sizeof(commit->encoded_reveal)) < 0) {
+    log_err(LD_DIR, "SR: Unable to encode our reveal value!");
+    goto error;
+  }
+
+  /* Now let's create the commitment */
+  tor_assert(commit->alg == SR_DIGEST_ALG);
+  /* The invariant length is used here since the encoded reveal variable
+   * has an extra byte added for the NULL terminated byte. */
+  if (crypto_digest256(commit->hashed_reveal, commit->encoded_reveal,
+                       SR_REVEAL_BASE64_LEN, commit->alg) < 0) {
+    goto error;
+  }
+
+  /* Now get the base64 blob that corresponds to our commit. */
+  if (commit_encode(commit, commit->encoded_commit,
+                    sizeof(commit->encoded_commit)) < 0) {
+    log_err(LD_DIR, "SR: Unable to encode our commit value!");
+    goto error;
+  }
+
+  log_debug(LD_DIR, "SR: Generated our commitment:");
+  commit_log(commit);
+  /* Our commit better be valid :). */
+  commit->valid = 1;
+  return commit;
+
+ error:
+  sr_commit_free(commit);
+  return NULL;
+}
+
+/* Compute the shared random value based on the active commits in our state. */
+void
+sr_compute_srv(void)
+{
+  uint64_t reveal_num = 0;
+  char *reveals = NULL;
+  smartlist_t *chunks, *commits;
+  digestmap_t *state_commits;
+
+  /* Computing a shared random value in the commit phase is very wrong. This
+   * should only happen at the very end of the reveal phase when a new
+   * protocol run is about to start. */
+  tor_assert(sr_state_get_phase() == SR_PHASE_REVEAL);
+  state_commits = sr_state_get_commits();
+
+  commits = smartlist_new();
+  chunks = smartlist_new();
+
+  /* We must make a list of commit ordered by authority fingerprint in
+   * ascending order as specified by proposal 250. */
+  DIGESTMAP_FOREACH(state_commits, key, sr_commit_t *, c) {
+    /* Extra safety net, make sure we have valid commit before using it. */
+    ASSERT_COMMIT_VALID(c);
+    /* Let's not use a commit from an authority that we don't know. It's
+     * possible that an authority could be removed during a protocol run so
+     * that commit value should never be used in the SRV computation. */
+    if (trusteddirserver_get_by_v3_auth_digest(c->rsa_identity) == NULL) {
+      log_warn(LD_DIR, "SR: Fingerprint %s is not from a recognized "
+               "authority. Discarding commit for the SRV computation.",
+               sr_commit_get_rsa_fpr(c));
+      continue;
+    }
+    /* We consider this commit valid. */
+    smartlist_add(commits, c);
+  } DIGESTMAP_FOREACH_END;
+  smartlist_sort(commits, compare_reveal_);
+
+  /* Now for each commit for that sorted list in ascending order, we'll
+   * build the element for each authority that needs to go into the srv
+   * computation. */
+  SMARTLIST_FOREACH_BEGIN(commits, const sr_commit_t *, c) {
+    char *element = get_srv_element_from_commit(c);
+    if (element) {
+      smartlist_add(chunks, element);
+      reveal_num++;
+    }
+  } SMARTLIST_FOREACH_END(c);
+  smartlist_free(commits);
+
+  {
+    /* Join all reveal values into one giant string that we'll hash so we
+     * can generated our shared random value. */
+    sr_srv_t *current_srv;
+    char hashed_reveals[DIGEST256_LEN];
+    reveals = smartlist_join_strings(chunks, "", 0, NULL);
+    SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+    smartlist_free(chunks);
+    if (crypto_digest256(hashed_reveals, reveals, strlen(reveals),
+                         SR_DIGEST_ALG) < 0) {
+      goto end;
+    }
+    current_srv = generate_srv(hashed_reveals, reveal_num,
+                               sr_state_get_previous_srv());
+    sr_state_set_current_srv(current_srv);
+    /* We have a fresh SRV, flag our state. */
+    sr_state_set_fresh_srv();
+  }
+
+ end:
+  tor_free(reveals);
+}
+
+/* Parse a commit from a vote or from our disk state and return a newly
+ * allocated commit object. NULL is returned on error.
+ *
+ * The commit's data is in <b>args</b> and the order matters very much:
+ *  version, algname, RSA fingerprint, commit value[, reveal value]
+ */
+sr_commit_t *
+sr_parse_commit(const smartlist_t *args)
+{
+  uint32_t version;
+  char *value, digest[DIGEST_LEN];
+  digest_algorithm_t alg;
+  const char *rsa_identity_fpr;
+  sr_commit_t *commit = NULL;
+
+  if (smartlist_len(args) < 4) {
+    goto error;
+  }
+
+  /* First is the version number of the SR protocol which indicates at which
+   * version that commit was created. */
+  value = smartlist_get(args, 0);
+  version = (uint32_t) tor_parse_ulong(value, 10, 1, UINT32_MAX, NULL, NULL);
+  if (version > SR_PROTO_VERSION) {
+    log_info(LD_DIR, "SR: Commit version %" PRIu32 " (%s) is not supported.",
+             version, escaped(value));
+    goto error;
+  }
+
+  /* Second is the algorithm. */
+  value = smartlist_get(args, 1);
+  alg = crypto_digest_algorithm_parse_name(value);
+  if (alg != SR_DIGEST_ALG) {
+    log_warn(LD_BUG, "SR: Commit algorithm %s is not recognized.",
+             escaped(value));
+    goto error;
+  }
+
+  /* Third argument is the RSA fingerprint of the auth and turn it into a
+   * digest value. */
+  rsa_identity_fpr = smartlist_get(args, 2);
+  if (base16_decode(digest, DIGEST_LEN, rsa_identity_fpr,
+                    HEX_DIGEST_LEN) < 0) {
+    log_warn(LD_DIR, "SR: RSA fingerprint %s not decodable",
+             escaped(rsa_identity_fpr));
+    goto error;
+  }
+
+  /* Allocate commit since we have a valid identity now. */
+  commit = commit_new(digest);
+
+  /* Fourth argument is the commitment value base64-encoded. */
+  value = smartlist_get(args, 3);
+  if (commit_decode(value, commit) < 0) {
+    goto error;
+  }
+
+  /* (Optional) Fifth argument is the revealed value. */
+  if (smartlist_len(args) > 4) {
+    value = smartlist_get(args, 4);
+    if (reveal_decode(value, commit) < 0) {
+      goto error;
+    }
+  }
+
+  return commit;
+
+ error:
+  sr_commit_free(commit);
+  return NULL;
+}
+
+/* Called when we are done parsing a vote by <b>voter_key</b> that might
+ * contain some useful <b>commits</b>. Find if any of them should be kept
+ * and update our state accordingly. Once done, the list of commitments will
+ * be empty. */
+void
+sr_handle_received_commits(smartlist_t *commits, crypto_pk_t *voter_key)
+{
+  char rsa_identity[DIGEST_LEN];
+
+  tor_assert(voter_key);
+
+  /* It's possible that the vote has _NO_ commits. */
+  if (commits == NULL) {
+    return;
+  }
+
+  /* Get the RSA identity fingerprint of this voter */
+  if (crypto_pk_get_digest(voter_key, rsa_identity) < 0) {
+    return;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(commits, sr_commit_t *, commit) {
+    /* We won't need the commit in this list anymore, kept or not. */
+    SMARTLIST_DEL_CURRENT(commits, commit);
+    /* Check if this commit is valid and should be stored in our state. */
+    if (!should_keep_commit(commit, rsa_identity,
+                            sr_state_get_phase())) {
+      sr_commit_free(commit);
+      continue;
+    }
+    /* Ok, we have a valid commit now that we are about to put in our state.
+     * so flag it valid from now on. */
+    commit->valid = 1;
+    /* Everything lines up: save this commit to state then! */
+    save_commit_to_state(commit);
+  } SMARTLIST_FOREACH_END(commit);
+}
+
+/* Return a heap-allocated string containing commits that should be put in
+ * the votes. It's the responsibility of the caller to free the string.
+ * This always return a valid string, either empty or with line(s). */
+char *
+sr_get_string_for_vote(void)
+{
+  char *vote_str = NULL;
+  digestmap_t *state_commits;
+  smartlist_t *chunks = smartlist_new();
+  const or_options_t *options = get_options();
+
+  /* Are we participating in the protocol? */
+  if (!options->AuthDirSharedRandomness) {
+    goto end;
+  }
+
+  log_debug(LD_DIR, "SR: Preparing our vote info:");
+
+  /* First line, put in the vote the participation flag. */
+  {
+    char *sr_flag_line;
+    tor_asprintf(&sr_flag_line, "%s\n", sr_flag_ns_str);
+    smartlist_add(chunks, sr_flag_line);
+  }
+
+  /* In our vote we include every commitment in our permanent state. */
+  state_commits = sr_state_get_commits();
+  smartlist_t *state_commit_vote_lines = smartlist_new();
+  DIGESTMAP_FOREACH(state_commits, key, const sr_commit_t *, commit) {
+    char *line = get_vote_line_from_commit(commit, sr_state_get_phase());
+    smartlist_add(state_commit_vote_lines, line);
+  } DIGESTMAP_FOREACH_END;
+
+  /* Sort the commit strings by version (string, not numeric), algorithm,
+   * and fingerprint. This makes sure the commit lines in votes are in a
+   * recognisable, stable order. */
+  smartlist_sort_strings(state_commit_vote_lines);
+
+  /* Now add the sorted list of commits to the vote */
+  smartlist_add_all(chunks, state_commit_vote_lines);
+  smartlist_free(state_commit_vote_lines);
+
+  /* Add the SRV value(s) if any. */
+  {
+    char *srv_lines = get_ns_str_from_sr_values(sr_state_get_previous_srv(),
+                                                sr_state_get_current_srv());
+    if (srv_lines) {
+      smartlist_add(chunks, srv_lines);
+    }
+  }
+
+ end:
+  vote_str = smartlist_join_strings(chunks, "", 0, NULL);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+  return vote_str;
+}
+
+/* Return a heap-allocated string that should be put in the consensus and
+ * contains the shared randomness values. It's the responsibility of the
+ * caller to free the string. NULL is returned if no SRV(s) available.
+ *
+ * This is called when a consensus (any flavor) is bring created thus it
+ * should NEVER change the state nor the state should be changed in between
+ * consensus creation.
+ *
+ * <b>num_srv_agreements</b> is taken from the votes thus the voted value
+ * that should be used.
+ * */
+char *
+sr_get_string_for_consensus(const smartlist_t *votes,
+                            int32_t num_srv_agreements)
+{
+  char *srv_str;
+  const or_options_t *options = get_options();
+
+  tor_assert(votes);
+
+  /* Not participating, avoid returning anything. */
+  if (!options->AuthDirSharedRandomness) {
+    log_info(LD_DIR, "SR: Support disabled (AuthDirSharedRandomness %d)",
+             options->AuthDirSharedRandomness);
+    goto end;
+  }
+
+  /* Set the global value of AuthDirNumSRVAgreements found in the votes. */
+  num_srv_agreements_from_vote = num_srv_agreements;
+
+  /* Check the votes and figure out if SRVs should be included in the final
+   * consensus. */
+  sr_srv_t *prev_srv = get_majority_srv_from_votes(votes, 0);
+  sr_srv_t *cur_srv = get_majority_srv_from_votes(votes, 1);
+  srv_str = get_ns_str_from_sr_values(prev_srv, cur_srv);
+  if (!srv_str) {
+    goto end;
+  }
+
+  return srv_str;
+ end:
+  return NULL;
+}
+
+/* We just computed a new <b>consensus</b>. Update our state with the SRVs
+ * from the consensus (might be NULL as well). Register the SRVs in our SR
+ * state and prepare for the upcoming protocol round. */
+void
+sr_act_post_consensus(const networkstatus_t *consensus)
+{
+  const or_options_t *options = get_options();
+
+  /* Don't act if our state hasn't been initialized. We can be called during
+   * boot time when loading consensus from disk which is prior to the
+   * initialization of the SR subsystem. We also should not be doing
+   * anything if we are _not_ a directory authority and if we are a bridge
+   * authority. */
+  if (!sr_state_is_initialized() || !authdir_mode_v3(options) ||
+      authdir_mode_bridge(options)) {
+    return;
+  }
+
+  /* Set the majority voted SRVs in our state even if both are NULL. It
+   * doesn't matter this is what the majority has decided. Obviously, we can
+   * only do that if we have a consensus. */
+  if (consensus) {
+    /* Start by freeing the current SRVs since the SRVs we believed during
+     * voting do not really matter. Now that all the votes are in, we use the
+     * majority's opinion on which are the active SRVs. */
+    sr_state_clean_srvs();
+    /* Reset the fresh flag of the SRV so we know that from now on we don't
+     * have a new SRV to vote for. We just used the one from the consensus
+     * decided by the majority. */
+    sr_state_unset_fresh_srv();
+    /* Set the SR values from the given consensus. */
+    sr_state_set_previous_srv(srv_dup(consensus->sr_info.previous_srv));
+    sr_state_set_current_srv(srv_dup(consensus->sr_info.current_srv));
+  }
+
+  /* Prepare our state so that it's ready for the next voting period. */
+  sr_state_update(voting_schedule_get_next_valid_after_time());
+}
+
+/* Initialize shared random subsystem. This MUST be called early in the boot
+ * process of tor. Return 0 on success else -1 on error. */
+int
+sr_init(int save_to_disk)
+{
+  return sr_state_init(save_to_disk, 1);
+}
+
+/* Save our state to disk and cleanup everything. */
+void
+sr_save_and_cleanup(void)
+{
+  sr_state_save();
+  sr_cleanup();
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/* Set the global value of number of SRV agreements so the test can play
+ * along by calling specific functions that don't parse the votes prior for
+ * the AuthDirNumSRVAgreements value. */
+void
+set_num_srv_agreements(int32_t value)
+{
+  num_srv_agreements_from_vote = value;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h
new file mode 100644
index 0000000000..93bab99f71
--- /dev/null
+++ b/src/feature/dirauth/shared_random.h
@@ -0,0 +1,194 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_SHARED_RANDOM_H
+#define TOR_SHARED_RANDOM_H
+
+/*
+ * This file contains ABI/API of the shared random protocol defined in
+ * proposal #250. Every public functions and data structure are namespaced
+ * with "sr_" which stands for shared random.
+ */
+
+#include "or/or.h"
+
+/* Protocol version */
+#define SR_PROTO_VERSION  1
+/* Default digest algorithm. */
+#define SR_DIGEST_ALG DIGEST_SHA3_256
+/* Invariant token in the SRV calculation. */
+#define SR_SRV_TOKEN "shared-random"
+/* Don't count the NUL terminated byte even though the TOKEN has it. */
+#define SR_SRV_TOKEN_LEN (sizeof(SR_SRV_TOKEN) - 1)
+
+/* Length of the random number (in bytes). */
+#define SR_RANDOM_NUMBER_LEN 32
+/* Size of a decoded commit value in a vote or state. It's a hash and a
+ * timestamp. It adds up to 40 bytes. */
+#define SR_COMMIT_LEN (sizeof(uint64_t) + DIGEST256_LEN)
+/* Size of a decoded reveal value from a vote or state. It's a 64 bit
+ * timestamp and the hashed random number. This adds up to 40 bytes. */
+#define SR_REVEAL_LEN (sizeof(uint64_t) + DIGEST256_LEN)
+/* Size of SRV message length. The construction is has follow:
+ *  "shared-random" | INT_8(reveal_num) | INT_4(version) | PREV_SRV */
+#define SR_SRV_MSG_LEN \
+  (SR_SRV_TOKEN_LEN + sizeof(uint64_t) + sizeof(uint32_t) + DIGEST256_LEN)
+
+/* Length of base64 encoded commit NOT including the NUL terminated byte.
+ * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
+#define SR_COMMIT_BASE64_LEN (BASE64_LEN(SR_COMMIT_LEN))
+/* Length of base64 encoded reveal NOT including the NUL terminated byte.
+ * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
+#define SR_REVEAL_BASE64_LEN (BASE64_LEN(SR_REVEAL_LEN))
+/* Length of base64 encoded shared random value. It's 32 bytes long so 44
+ * bytes from the base64_encode_size formula. That includes the '='
+ * character at the end. */
+#define SR_SRV_VALUE_BASE64_LEN (BASE64_LEN(DIGEST256_LEN))
+
+/* Assert if commit valid flag is not set. */
+#define ASSERT_COMMIT_VALID(c) tor_assert((c)->valid)
+
+/* Protocol phase. */
+typedef enum {
+  /* Commitment phase */
+  SR_PHASE_COMMIT  = 1,
+  /* Reveal phase */
+  SR_PHASE_REVEAL  = 2,
+} sr_phase_t;
+
+/* A shared random value (SRV). */
+typedef struct sr_srv_t {
+  /* The number of reveal values used to derive this SRV. */
+  uint64_t num_reveals;
+  /* The actual value. This is the stored result of SHA3-256. */
+  uint8_t value[DIGEST256_LEN];
+} sr_srv_t;
+
+/* A commit (either ours or from another authority). */
+typedef struct sr_commit_t {
+  /* Hashing algorithm used. */
+  digest_algorithm_t alg;
+  /* Indicate if this commit has been verified thus valid. */
+  unsigned int valid:1;
+
+  /* Commit owner info */
+
+  /* The RSA identity key of the authority and its base16 representation,
+   * which includes the NUL terminated byte. */
+  char rsa_identity[DIGEST_LEN];
+  char rsa_identity_hex[HEX_DIGEST_LEN + 1];
+
+  /* Commitment information */
+
+  /* Timestamp of reveal. Correspond to TIMESTAMP. */
+  uint64_t reveal_ts;
+  /* H(REVEAL) as found in COMMIT message. */
+  char hashed_reveal[DIGEST256_LEN];
+  /* Base64 encoded COMMIT. We use this to put it in our vote. */
+  char encoded_commit[SR_COMMIT_BASE64_LEN + 1];
+
+  /* Reveal information */
+
+  /* H(RN) which is what we used as the random value for this commit. We
+   * don't use the raw bytes since those are sent on the network thus
+   * avoiding possible information leaks of our PRNG. */
+  uint8_t random_number[SR_RANDOM_NUMBER_LEN];
+  /* Timestamp of commit. Correspond to TIMESTAMP. */
+  uint64_t commit_ts;
+  /* This is the whole reveal message. We use it during verification */
+  char encoded_reveal[SR_REVEAL_BASE64_LEN + 1];
+} sr_commit_t;
+
+/* API */
+
+/* Public methods used _outside_ of the module.
+ *
+ * We need to nullify them if the module is disabled. */
+#ifdef HAVE_MODULE_DIRAUTH
+
+int sr_init(int save_to_disk);
+void sr_save_and_cleanup(void);
+void sr_act_post_consensus(const networkstatus_t *consensus);
+
+#else /* HAVE_MODULE_DIRAUTH */
+
+static inline int
+sr_init(int save_to_disk)
+{
+  (void) save_to_disk;
+  /* Always return success. */
+  return 0;
+}
+
+static inline void
+sr_save_and_cleanup(void)
+{
+}
+
+static inline void
+sr_act_post_consensus(const networkstatus_t *consensus)
+{
+  (void) consensus;
+}
+
+#endif /* HAVE_MODULE_DIRAUTH */
+
+/* Public methods used only by dirauth code. */
+
+void sr_handle_received_commits(smartlist_t *commits,
+                                crypto_pk_t *voter_key);
+sr_commit_t *sr_parse_commit(const smartlist_t *args);
+char *sr_get_string_for_vote(void);
+char *sr_get_string_for_consensus(const smartlist_t *votes,
+                                  int32_t num_srv_agreements);
+void sr_commit_free_(sr_commit_t *commit);
+#define sr_commit_free(sr) FREE_AND_NULL(sr_commit_t, sr_commit_free_, (sr))
+
+/* Private methods (only used by shared_random_state.c): */
+static inline
+const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit)
+{
+  return commit->rsa_identity_hex;
+}
+
+void sr_compute_srv(void);
+sr_commit_t *sr_generate_our_commit(time_t timestamp,
+                                    const authority_cert_t *my_rsa_cert);
+
+#ifdef SHARED_RANDOM_PRIVATE
+
+/* Encode */
+STATIC int reveal_encode(const sr_commit_t *commit, char *dst, size_t len);
+STATIC int commit_encode(const sr_commit_t *commit, char *dst, size_t len);
+/* Decode. */
+STATIC int commit_decode(const char *encoded, sr_commit_t *commit);
+STATIC int reveal_decode(const char *encoded, sr_commit_t *commit);
+
+STATIC int commit_has_reveal_value(const sr_commit_t *commit);
+
+STATIC int verify_commit_and_reveal(const sr_commit_t *commit);
+
+STATIC sr_srv_t *get_majority_srv_from_votes(const smartlist_t *votes,
+                                             int current);
+
+STATIC void save_commit_to_state(sr_commit_t *commit);
+STATIC sr_srv_t *srv_dup(const sr_srv_t *orig);
+STATIC int commitments_are_the_same(const sr_commit_t *commit_one,
+                                    const sr_commit_t *commit_two);
+STATIC int commit_is_authoritative(const sr_commit_t *commit,
+                                   const char *voter_key);
+STATIC int should_keep_commit(const sr_commit_t *commit,
+                              const char *voter_key,
+                              sr_phase_t phase);
+STATIC void save_commit_during_reveal_phase(const sr_commit_t *commit);
+
+#endif /* defined(SHARED_RANDOM_PRIVATE) */
+
+#ifdef TOR_UNIT_TESTS
+
+void set_num_srv_agreements(int32_t value);
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* !defined(TOR_SHARED_RANDOM_H) */
+
diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c
new file mode 100644
index 0000000000..87ddcc0736
--- /dev/null
+++ b/src/feature/dirauth/shared_random_state.c
@@ -0,0 +1,1326 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shared_random_state.c
+ *
+ * \brief Functions and data structures for the state of the random protocol
+ *        as defined in proposal #250.
+ **/
+
+#define SHARED_RANDOM_STATE_PRIVATE
+
+#include "or/or.h"
+#include "or/config.h"
+#include "or/confparse.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/dirauth/dirvote.h"
+#include "or/networkstatus.h"
+#include "or/router.h"
+#include "or/dirauth/shared_random.h"
+#include "or/shared_random_client.h"
+#include "or/dirauth/shared_random_state.h"
+#include "or/voting_schedule.h"
+#include "lib/encoding/confline.h"
+
+#include "or/or_state_st.h"
+
+/* Default filename of the shared random state on disk. */
+static const char default_fname[] = "sr-state";
+
+/* String representation of a protocol phase. */
+static const char *phase_str[] = { "unknown", "commit", "reveal" };
+
+/* Our shared random protocol state. There is only one possible state per
+ * protocol run so this is the global state which is reset at every run once
+ * the shared random value has been computed. */
+static sr_state_t *sr_state = NULL;
+
+/* Representation of our persistent state on disk. The sr_state above
+ * contains the data parsed from this state. When we save to disk, we
+ * translate the sr_state to this sr_disk_state. */
+static sr_disk_state_t *sr_disk_state = NULL;
+
+/* Disk state file keys. */
+static const char dstate_commit_key[] = "Commit";
+static const char dstate_prev_srv_key[] = "SharedRandPreviousValue";
+static const char dstate_cur_srv_key[] = "SharedRandCurrentValue";
+
+/** dummy instance of sr_disk_state_t, used for type-checking its
+ * members with CONF_CHECK_VAR_TYPE. */
+DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t);
+
+/* These next two are duplicates or near-duplicates from config.c */
+#define VAR(name, conftype, member, initvalue)                              \
+  { name, CONFIG_TYPE_ ## conftype, offsetof(sr_disk_state_t, member),      \
+      initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) }
+/* As VAR, but the option name and member name are the same. */
+#define V(member, conftype, initvalue) \
+  VAR(#member, conftype, member, initvalue)
+/* Our persistent state magic number. */
+#define SR_DISK_STATE_MAGIC 0x98AB1254
+
+static int
+disk_state_validate_cb(void *old_state, void *state, void *default_state,
+                       int from_setconf, char **msg);
+
+/* Array of variables that are saved to disk as a persistent state. */
+static config_var_t state_vars[] = {
+  V(Version,                    UINT, "0"),
+  V(TorVersion,                 STRING, NULL),
+  V(ValidAfter,                 ISOTIME, NULL),
+  V(ValidUntil,                 ISOTIME, NULL),
+
+  V(Commit,                     LINELIST, NULL),
+
+  V(SharedRandValues,           LINELIST_V, NULL),
+  VAR("SharedRandPreviousValue",LINELIST_S, SharedRandValues, NULL),
+  VAR("SharedRandCurrentValue", LINELIST_S, SharedRandValues, NULL),
+  END_OF_CONFIG_VARS
+};
+
+/* "Extra" variable in the state that receives lines we can't parse. This
+ * lets us preserve options from versions of Tor newer than us. */
+static config_var_t state_extra_var = {
+  "__extra", CONFIG_TYPE_LINELIST,
+  offsetof(sr_disk_state_t, ExtraLines), NULL
+  CONF_TEST_MEMBERS(sr_disk_state_t, LINELIST, ExtraLines)
+};
+
+/* Configuration format of sr_disk_state_t. */
+static const config_format_t state_format = {
+  sizeof(sr_disk_state_t),
+  SR_DISK_STATE_MAGIC,
+  offsetof(sr_disk_state_t, magic_),
+  NULL,
+  NULL,
+  state_vars,
+  disk_state_validate_cb,
+  &state_extra_var,
+};
+
+/* Return a string representation of a protocol phase. */
+STATIC const char *
+get_phase_str(sr_phase_t phase)
+{
+  const char *the_string = NULL;
+
+  switch (phase) {
+  case SR_PHASE_COMMIT:
+  case SR_PHASE_REVEAL:
+    the_string = phase_str[phase];
+    break;
+  default:
+    /* Unknown phase shouldn't be possible. */
+    tor_assert(0);
+  }
+
+  return the_string;
+}
+/* Return the time we should expire the state file created at <b>now</b>.
+ * We expire the state file in the beginning of the next protocol run. */
+STATIC time_t
+get_state_valid_until_time(time_t now)
+{
+  int total_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
+  int current_round, voting_interval, rounds_left;
+  time_t valid_until, beginning_of_current_round;
+
+  voting_interval = get_voting_interval();
+  /* Find the time the current round started. */
+  beginning_of_current_round = get_start_time_of_current_round();
+
+  /* Find how many rounds are left till the end of the protocol run */
+  current_round = (now / voting_interval) % total_rounds;
+  rounds_left = total_rounds - current_round;
+
+  /* To find the valid-until time now, take the start time of the current
+   * round and add to it the time it takes for the leftover rounds to
+   * complete. */
+  valid_until = beginning_of_current_round + (rounds_left * voting_interval);
+
+  { /* Logging */
+    char tbuf[ISO_TIME_LEN + 1];
+    format_iso_time(tbuf, valid_until);
+    log_debug(LD_DIR, "SR: Valid until time for state set to %s.", tbuf);
+  }
+
+  return valid_until;
+}
+
+/* Given the consensus 'valid-after' time, return the protocol phase we should
+ * be in. */
+STATIC sr_phase_t
+get_sr_protocol_phase(time_t valid_after)
+{
+  /* Shared random protocol has two phases, commit and reveal. */
+  int total_periods = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
+  int current_slot;
+
+  /* Split time into slots of size 'voting_interval'. See which slot we are
+   * currently into, and find which phase it corresponds to. */
+  current_slot = (valid_after / get_voting_interval()) % total_periods;
+
+  if (current_slot < SHARED_RANDOM_N_ROUNDS) {
+    return SR_PHASE_COMMIT;
+  } else {
+    return SR_PHASE_REVEAL;
+  }
+}
+
+/* Add the given <b>commit</b> to <b>state</b>. It MUST be a valid commit
+ * and there shouldn't be a commit from the same authority in the state
+ * already else verification hasn't been done prior. This takes ownership of
+ * the commit once in our state. */
+static void
+commit_add_to_state(sr_commit_t *commit, sr_state_t *state)
+{
+  sr_commit_t *saved_commit;
+
+  tor_assert(commit);
+  tor_assert(state);
+
+  saved_commit = digestmap_set(state->commits, commit->rsa_identity,
+                               commit);
+  if (saved_commit != NULL) {
+    /* This means we already have that commit in our state so adding twice
+     * the same commit is either a code flow error, a corrupted disk state
+     * or some new unknown issue. */
+    log_warn(LD_DIR, "SR: Commit from %s exists in our state while "
+                     "adding it: '%s'", sr_commit_get_rsa_fpr(commit),
+                     commit->encoded_commit);
+    sr_commit_free(saved_commit);
+  }
+}
+
+/* Helper: deallocate a commit object. (Used with digestmap_free(), which
+ * requires a function pointer whose argument is void *). */
+static void
+commit_free_(void *p)
+{
+  sr_commit_free_(p);
+}
+
+#define state_free(val) \
+  FREE_AND_NULL(sr_state_t, state_free_, (val))
+
+/* Free a state that was allocated with state_new(). */
+static void
+state_free_(sr_state_t *state)
+{
+  if (state == NULL) {
+    return;
+  }
+  tor_free(state->fname);
+  digestmap_free(state->commits, commit_free_);
+  tor_free(state->current_srv);
+  tor_free(state->previous_srv);
+  tor_free(state);
+}
+
+/* Allocate an sr_state_t object and returns it. If no <b>fname</b>, the
+ * default file name is used. This function does NOT initialize the state
+ * timestamp, phase or shared random value. NULL is never returned. */
+static sr_state_t *
+state_new(const char *fname, time_t now)
+{
+  sr_state_t *new_state = tor_malloc_zero(sizeof(*new_state));
+  /* If file name is not provided, use default. */
+  if (fname == NULL) {
+    fname = default_fname;
+  }
+  new_state->fname = tor_strdup(fname);
+  new_state->version = SR_PROTO_VERSION;
+  new_state->commits = digestmap_new();
+  new_state->phase = get_sr_protocol_phase(now);
+  new_state->valid_until = get_state_valid_until_time(now);
+  return new_state;
+}
+
+/* Set our global state pointer with the one given. */
+static void
+state_set(sr_state_t *state)
+{
+  tor_assert(state);
+  if (sr_state != NULL) {
+    state_free(sr_state);
+  }
+  sr_state = state;
+}
+
+#define disk_state_free(val) \
+  FREE_AND_NULL(sr_disk_state_t, disk_state_free_, (val))
+
+/* Free an allocated disk state. */
+static void
+disk_state_free_(sr_disk_state_t *state)
+{
+  if (state == NULL) {
+    return;
+  }
+  config_free(&state_format, state);
+}
+
+/* Allocate a new disk state, initialize it and return it. */
+static sr_disk_state_t *
+disk_state_new(time_t now)
+{
+  sr_disk_state_t *new_state = tor_malloc_zero(sizeof(*new_state));
+
+  new_state->magic_ = SR_DISK_STATE_MAGIC;
+  new_state->Version = SR_PROTO_VERSION;
+  new_state->TorVersion = tor_strdup(get_version());
+  new_state->ValidUntil = get_state_valid_until_time(now);
+  new_state->ValidAfter = now;
+
+  /* Init config format. */
+  config_init(&state_format, new_state);
+  return new_state;
+}
+
+/* Set our global disk state with the given state. */
+static void
+disk_state_set(sr_disk_state_t *state)
+{
+  tor_assert(state);
+  if (sr_disk_state != NULL) {
+    disk_state_free(sr_disk_state);
+  }
+  sr_disk_state = state;
+}
+
+/* Return -1 if the disk state is invalid (something in there that we can't or
+ * shouldn't use). Return 0 if everything checks out. */
+static int
+disk_state_validate(const sr_disk_state_t *state)
+{
+  time_t now;
+
+  tor_assert(state);
+
+  /* Do we support the protocol version in the state or is it 0 meaning
+   * Version wasn't found in the state file or bad anyway ? */
+  if (state->Version == 0 || state->Version > SR_PROTO_VERSION) {
+    goto invalid;
+  }
+
+  /* If the valid until time is before now, we shouldn't use that state. */
+  now = time(NULL);
+  if (state->ValidUntil < now) {
+    log_info(LD_DIR, "SR: Disk state has expired. Ignoring it.");
+    goto invalid;
+  }
+
+  /* Make sure we don't have a valid after time that is earlier than a valid
+   * until time which would make things not work well. */
+  if (state->ValidAfter >= state->ValidUntil) {
+    log_info(LD_DIR, "SR: Disk state valid after/until times are invalid.");
+    goto invalid;
+  }
+
+  return 0;
+
+ invalid:
+  return -1;
+}
+
+/* Validate the disk state (NOP for now). */
+static int
+disk_state_validate_cb(void *old_state, void *state, void *default_state,
+                       int from_setconf, char **msg)
+{
+  /* We don't use these; only options do. */
+  (void) from_setconf;
+  (void) default_state;
+  (void) old_state;
+
+  /* This is called by config_dump which is just before we are about to
+   * write it to disk. At that point, our global memory state has been
+   * copied to the disk state so it's fair to assume it's trustable. */
+  (void) state;
+  (void) msg;
+  return 0;
+}
+
+/* Parse the Commit line(s) in the disk state and translate them to the
+ * the memory state. Return 0 on success else -1 on error. */
+static int
+disk_state_parse_commits(sr_state_t *state,
+                         const sr_disk_state_t *disk_state)
+{
+  config_line_t *line;
+  smartlist_t *args = NULL;
+
+  tor_assert(state);
+  tor_assert(disk_state);
+
+  for (line = disk_state->Commit; line; line = line->next) {
+    sr_commit_t *commit = NULL;
+
+    /* Extra safety. */
+    if (strcasecmp(line->key, dstate_commit_key) ||
+        line->value == NULL) {
+      /* Ignore any lines that are not commits. */
+      tor_fragile_assert();
+      continue;
+    }
+    args = smartlist_new();
+    smartlist_split_string(args, line->value, " ",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+    if (smartlist_len(args) < 3) {
+      log_warn(LD_BUG, "SR: Too few arguments in Commit Line: %s",
+               escaped(line->value));
+      goto error;
+    }
+    commit = sr_parse_commit(args);
+    if (commit == NULL) {
+      /* Ignore badly formed commit. It could also be a authority
+       * fingerprint that we don't know about so it shouldn't be used. */
+      smartlist_free(args);
+      continue;
+    }
+    /* We consider parseable commit from our disk state to be valid because
+     * they need to be in the first place to get in there. */
+    commit->valid = 1;
+    /* Add commit to our state pointer. */
+    commit_add_to_state(commit, state);
+
+    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+    smartlist_free(args);
+  }
+
+  return 0;
+
+ error:
+  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
+  smartlist_free(args);
+  return -1;
+}
+
+/* Parse a share random value line from the disk state and save it to dst
+ * which is an allocated srv object. Return 0 on success else -1. */
+static int
+disk_state_parse_srv(const char *value, sr_srv_t *dst)
+{
+  int ret = -1;
+  smartlist_t *args;
+  sr_srv_t *srv;
+
+  tor_assert(value);
+  tor_assert(dst);
+
+  args = smartlist_new();
+  smartlist_split_string(args, value, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  if (smartlist_len(args) < 2) {
+    log_warn(LD_BUG, "SR: Too few arguments in shared random value. "
+             "Line: %s", escaped(value));
+    goto error;
+  }
+  srv = sr_parse_srv(args);
+  if (srv == NULL) {
+    goto error;
+  }
+  dst->num_reveals = srv->num_reveals;
+  memcpy(dst->value, srv->value, sizeof(dst->value));
+  tor_free(srv);
+  ret = 0;
+
+ error:
+  SMARTLIST_FOREACH(args, char *, s, tor_free(s));
+  smartlist_free(args);
+  return ret;
+}
+
+/* Parse both SharedRandCurrentValue and SharedRandPreviousValue line from
+ * the state. Return 0 on success else -1. */
+static int
+disk_state_parse_sr_values(sr_state_t *state,
+                           const sr_disk_state_t *disk_state)
+{
+  /* Only one value per type (current or previous) is allowed so we keep
+   * track of it with these flag. */
+  unsigned int seen_previous = 0, seen_current = 0;
+  config_line_t *line;
+  sr_srv_t *srv = NULL;
+
+  tor_assert(state);
+  tor_assert(disk_state);
+
+  for (line = disk_state->SharedRandValues; line; line = line->next) {
+    if (line->value == NULL) {
+      continue;
+    }
+    srv = tor_malloc_zero(sizeof(*srv));
+    if (disk_state_parse_srv(line->value, srv) < 0) {
+      log_warn(LD_BUG, "SR: Broken current SRV line in state %s",
+               escaped(line->value));
+      goto bad;
+    }
+    if (!strcasecmp(line->key, dstate_prev_srv_key)) {
+      if (seen_previous) {
+        log_warn(LD_DIR, "SR: Second previous SRV value seen. Bad state");
+        goto bad;
+      }
+      state->previous_srv = srv;
+      seen_previous = 1;
+    } else if (!strcasecmp(line->key, dstate_cur_srv_key)) {
+      if (seen_current) {
+        log_warn(LD_DIR, "SR: Second current SRV value seen. Bad state");
+        goto bad;
+      }
+      state->current_srv = srv;
+      seen_current = 1;
+    } else {
+      /* Unknown key. Ignoring. */
+      tor_free(srv);
+    }
+  }
+
+  return 0;
+ bad:
+  tor_free(srv);
+  return -1;
+}
+
+/* Parse the given disk state and set a newly allocated state. On success,
+ * return that state else NULL. */
+static sr_state_t *
+disk_state_parse(const sr_disk_state_t *new_disk_state)
+{
+  sr_state_t *new_state = state_new(default_fname, time(NULL));
+
+  tor_assert(new_disk_state);
+
+  new_state->version = new_disk_state->Version;
+  new_state->valid_until = new_disk_state->ValidUntil;
+  new_state->valid_after = new_disk_state->ValidAfter;
+
+  /* Set our current phase according to the valid-after time in our disk
+   * state. The disk state we are parsing contains everything for the phase
+   * starting at valid_after so make sure our phase reflects that. */
+  new_state->phase = get_sr_protocol_phase(new_state->valid_after);
+
+  /* Parse the shared random values. */
+  if (disk_state_parse_sr_values(new_state, new_disk_state) < 0) {
+    goto error;
+  }
+  /* Parse the commits. */
+  if (disk_state_parse_commits(new_state, new_disk_state) < 0) {
+    goto error;
+  }
+  /* Great! This new state contains everything we had on disk. */
+  return new_state;
+
+ error:
+  state_free(new_state);
+  return NULL;
+}
+
+/* From a valid commit object and an allocated config line, set the line's
+ * value to the state string representation of a commit. */
+static void
+disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line)
+{
+  char *reveal_str = NULL;
+
+  tor_assert(commit);
+  tor_assert(line);
+
+  if (!tor_mem_is_zero(commit->encoded_reveal,
+                       sizeof(commit->encoded_reveal))) {
+    /* Add extra whitespace so we can format the line correctly. */
+    tor_asprintf(&reveal_str, " %s", commit->encoded_reveal);
+  }
+  tor_asprintf(&line->value, "%u %s %s %s%s",
+               SR_PROTO_VERSION,
+               crypto_digest_algorithm_get_name(commit->alg),
+               sr_commit_get_rsa_fpr(commit),
+               commit->encoded_commit,
+               reveal_str != NULL ? reveal_str : "");
+  if (reveal_str != NULL) {
+    memwipe(reveal_str, 0, strlen(reveal_str));
+    tor_free(reveal_str);
+  }
+}
+
+/* From a valid srv object and an allocated config line, set the line's
+ * value to the state string representation of a shared random value. */
+static void
+disk_state_put_srv_line(const sr_srv_t *srv, config_line_t *line)
+{
+  char encoded[SR_SRV_VALUE_BASE64_LEN + 1];
+
+  tor_assert(line);
+
+  /* No SRV value thus don't add the line. This is possible since we might
+   * not have a current or previous SRV value in our state. */
+  if (srv == NULL) {
+    return;
+  }
+  sr_srv_encode(encoded, sizeof(encoded), srv);
+  tor_asprintf(&line->value, "%" PRIu64 " %s", srv->num_reveals, encoded);
+}
+
+/* Reset disk state that is free allocated memory and zeroed the object. */
+static void
+disk_state_reset(void)
+{
+  /* Free allocated memory */
+  config_free_lines(sr_disk_state->Commit);
+  config_free_lines(sr_disk_state->SharedRandValues);
+  config_free_lines(sr_disk_state->ExtraLines);
+  tor_free(sr_disk_state->TorVersion);
+
+  /* Clean up the struct */
+  memset(sr_disk_state, 0, sizeof(*sr_disk_state));
+
+  /* Reset it with useful data */
+  sr_disk_state->magic_ = SR_DISK_STATE_MAGIC;
+  sr_disk_state->TorVersion = tor_strdup(get_version());
+}
+
+/* Update our disk state based on our global SR state. */
+static void
+disk_state_update(void)
+{
+  config_line_t **next, *line;
+
+  tor_assert(sr_disk_state);
+  tor_assert(sr_state);
+
+  /* Reset current disk state. */
+  disk_state_reset();
+
+  /* First, update elements that we don't need to do a construction. */
+  sr_disk_state->Version = sr_state->version;
+  sr_disk_state->ValidUntil = sr_state->valid_until;
+  sr_disk_state->ValidAfter = sr_state->valid_after;
+
+  /* Shared random values. */
+  next = &sr_disk_state->SharedRandValues;
+  if (sr_state->previous_srv != NULL) {
+    *next = line = tor_malloc_zero(sizeof(config_line_t));
+    line->key = tor_strdup(dstate_prev_srv_key);
+    disk_state_put_srv_line(sr_state->previous_srv, line);
+    /* Go to the next shared random value. */
+    next = &(line->next);
+  }
+  if (sr_state->current_srv != NULL) {
+    *next = line = tor_malloc_zero(sizeof(*line));
+    line->key = tor_strdup(dstate_cur_srv_key);
+    disk_state_put_srv_line(sr_state->current_srv, line);
+  }
+
+  /* Parse the commits and construct config line(s). */
+  next = &sr_disk_state->Commit;
+  DIGESTMAP_FOREACH(sr_state->commits, key, sr_commit_t *, commit) {
+    *next = line = tor_malloc_zero(sizeof(*line));
+    line->key = tor_strdup(dstate_commit_key);
+    disk_state_put_commit_line(commit, line);
+    next = &(line->next);
+  } DIGESTMAP_FOREACH_END;
+}
+
+/* Load state from disk and put it into our disk state. If the state passes
+ * validation, our global state will be updated with it. Return 0 on
+ * success. On error, -EINVAL is returned if the state on disk did contained
+ * something malformed or is unreadable. -ENOENT is returned indicating that
+ * the state file is either empty of non existing. */
+static int
+disk_state_load_from_disk(void)
+{
+  int ret;
+  char *fname;
+
+  fname = get_datadir_fname(default_fname);
+  ret = disk_state_load_from_disk_impl(fname);
+  tor_free(fname);
+
+  return ret;
+}
+
+/* Helper for disk_state_load_from_disk(). */
+STATIC int
+disk_state_load_from_disk_impl(const char *fname)
+{
+  int ret;
+  char *content = NULL;
+  sr_state_t *parsed_state = NULL;
+  sr_disk_state_t *disk_state = NULL;
+
+  /* Read content of file so we can parse it. */
+  if ((content = read_file_to_str(fname, 0, NULL)) == NULL) {
+    log_warn(LD_FS, "SR: Unable to read SR state file %s",
+             escaped(fname));
+    ret = -errno;
+    goto error;
+  }
+
+  {
+    config_line_t *lines = NULL;
+    char *errmsg = NULL;
+
+    /* Every error in this code path will return EINVAL. */
+    ret = -EINVAL;
+    if (config_get_lines(content, &lines, 0) < 0) {
+      config_free_lines(lines);
+      goto error;
+    }
+
+    disk_state = disk_state_new(time(NULL));
+    config_assign(&state_format, disk_state, lines, 0, &errmsg);
+    config_free_lines(lines);
+    if (errmsg) {
+      log_warn(LD_DIR, "SR: Reading state error: %s", errmsg);
+      tor_free(errmsg);
+      goto error;
+    }
+  }
+
+  /* So far so good, we've loaded our state file into our disk state. Let's
+   * validate it and then parse it. */
+  if (disk_state_validate(disk_state) < 0) {
+    ret = -EINVAL;
+    goto error;
+  }
+
+  parsed_state = disk_state_parse(disk_state);
+  if (parsed_state == NULL) {
+    ret = -EINVAL;
+    goto error;
+  }
+  state_set(parsed_state);
+  disk_state_set(disk_state);
+  tor_free(content);
+  log_info(LD_DIR, "SR: State loaded successfully from file %s", fname);
+  return 0;
+
+ error:
+  disk_state_free(disk_state);
+  tor_free(content);
+  return ret;
+}
+
+/* Save the disk state to disk but before that update it from the current
+ * state so we always have the latest. Return 0 on success else -1. */
+static int
+disk_state_save_to_disk(void)
+{
+  int ret;
+  char *state, *content = NULL, *fname = NULL;
+  char tbuf[ISO_TIME_LEN + 1];
+  time_t now = time(NULL);
+
+  /* If we didn't have the opportunity to setup an internal disk state,
+   * don't bother saving something to disk. */
+  if (sr_disk_state == NULL) {
+    ret = 0;
+    goto done;
+  }
+
+  /* Make sure that our disk state is up to date with our memory state
+   * before saving it to disk. */
+  disk_state_update();
+  state = config_dump(&state_format, NULL, sr_disk_state, 0, 0);
+  format_local_iso_time(tbuf, now);
+  tor_asprintf(&content,
+               "# Tor shared random state file last generated on %s "
+               "local time\n"
+               "# Other times below are in UTC\n"
+               "# Please *do not* edit this file.\n\n%s",
+               tbuf, state);
+  tor_free(state);
+  fname = get_datadir_fname(default_fname);
+  if (write_str_to_file(fname, content, 0) < 0) {
+    log_warn(LD_FS, "SR: Unable to write SR state to file %s", fname);
+    ret = -1;
+    goto done;
+  }
+  ret = 0;
+  log_debug(LD_DIR, "SR: Saved state to file %s", fname);
+
+ done:
+  tor_free(fname);
+  tor_free(content);
+  return ret;
+}
+
+/* Reset our state to prepare for a new protocol run. Once this returns, all
+ * commits in the state will be removed and freed. */
+STATIC void
+reset_state_for_new_protocol_run(time_t valid_after)
+{
+  tor_assert(sr_state);
+
+  /* Keep counters in track */
+  sr_state->n_reveal_rounds = 0;
+  sr_state->n_commit_rounds = 0;
+  sr_state->n_protocol_runs++;
+
+  /* Reset valid-until */
+  sr_state->valid_until = get_state_valid_until_time(valid_after);
+  sr_state->valid_after = valid_after;
+
+  /* We are in a new protocol run so cleanup commits. */
+  sr_state_delete_commits();
+}
+
+/* This is the first round of the new protocol run starting at
+ * <b>valid_after</b>. Do the necessary housekeeping. */
+STATIC void
+new_protocol_run(time_t valid_after)
+{
+  sr_commit_t *our_commitment = NULL;
+
+  /* Only compute the srv at the end of the reveal phase. */
+  if (sr_state->phase == SR_PHASE_REVEAL) {
+    /* We are about to compute a new shared random value that will be set in
+     * our state as the current value so rotate values. */
+    state_rotate_srv();
+    /* Compute the shared randomness value of the day. */
+    sr_compute_srv();
+  }
+
+  /* Prepare for the new protocol run by reseting the state */
+  reset_state_for_new_protocol_run(valid_after);
+
+  /* Do some logging */
+  log_info(LD_DIR, "SR: Protocol run #%" PRIu64 " starting!",
+           sr_state->n_protocol_runs);
+
+  /* Generate fresh commitments for this protocol run */
+  our_commitment = sr_generate_our_commit(valid_after,
+                                          get_my_v3_authority_cert());
+  if (our_commitment) {
+    /* Add our commitment to our state. In case we are unable to create one
+     * (highly unlikely), we won't vote for this protocol run since our
+     * commitment won't be in our state. */
+    sr_state_add_commit(our_commitment);
+  }
+}
+
+/* Return 1 iff the <b>next_phase</b> is a phase transition from the current
+ * phase that is it's different. */
+STATIC int
+is_phase_transition(sr_phase_t next_phase)
+{
+  return sr_state->phase != next_phase;
+}
+
+/* Helper function: return a commit using the RSA fingerprint of the
+ * authority or NULL if no such commit is known. */
+static sr_commit_t *
+state_query_get_commit(const char *rsa_fpr)
+{
+  tor_assert(rsa_fpr);
+  return digestmap_get(sr_state->commits, rsa_fpr);
+}
+
+/* Helper function: This handles the GET state action using an
+ * <b>obj_type</b> and <b>data</b> needed for the action. */
+static void *
+state_query_get_(sr_state_object_t obj_type, const void *data)
+{
+  void *obj = NULL;
+
+  switch (obj_type) {
+  case SR_STATE_OBJ_COMMIT:
+  {
+    obj = state_query_get_commit(data);
+    break;
+  }
+  case SR_STATE_OBJ_COMMITS:
+    obj = sr_state->commits;
+    break;
+  case SR_STATE_OBJ_CURSRV:
+    obj = sr_state->current_srv;
+    break;
+  case SR_STATE_OBJ_PREVSRV:
+    obj = sr_state->previous_srv;
+    break;
+  case SR_STATE_OBJ_PHASE:
+    obj = &sr_state->phase;
+    break;
+  case SR_STATE_OBJ_VALID_AFTER:
+  default:
+    tor_assert(0);
+  }
+  return obj;
+}
+
+/* Helper function: This handles the PUT state action using an
+ * <b>obj_type</b> and <b>data</b> needed for the action. */
+static void
+state_query_put_(sr_state_object_t obj_type, void *data)
+{
+  switch (obj_type) {
+  case SR_STATE_OBJ_COMMIT:
+  {
+    sr_commit_t *commit = data;
+    tor_assert(commit);
+    commit_add_to_state(commit, sr_state);
+    break;
+  }
+  case SR_STATE_OBJ_CURSRV:
+    sr_state->current_srv = (sr_srv_t *) data;
+    break;
+  case SR_STATE_OBJ_PREVSRV:
+    sr_state->previous_srv = (sr_srv_t *) data;
+    break;
+  case SR_STATE_OBJ_VALID_AFTER:
+    sr_state->valid_after = *((time_t *) data);
+    break;
+  /* It's not allowed to change the phase nor the full commitments map from
+   * the state. The phase is decided during a strict process post voting and
+   * the commits should be put individually. */
+  case SR_STATE_OBJ_PHASE:
+  case SR_STATE_OBJ_COMMITS:
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Helper function: This handles the DEL_ALL state action using an
+ * <b>obj_type</b> and <b>data</b> needed for the action. */
+static void
+state_query_del_all_(sr_state_object_t obj_type)
+{
+  switch (obj_type) {
+  case SR_STATE_OBJ_COMMIT:
+  {
+    /* We are in a new protocol run so cleanup commitments. */
+    DIGESTMAP_FOREACH_MODIFY(sr_state->commits, key, sr_commit_t *, c) {
+      sr_commit_free(c);
+      MAP_DEL_CURRENT(key);
+    } DIGESTMAP_FOREACH_END;
+    break;
+  }
+  /* The following object are _NOT_ suppose to be removed. */
+  case SR_STATE_OBJ_CURSRV:
+  case SR_STATE_OBJ_PREVSRV:
+  case SR_STATE_OBJ_PHASE:
+  case SR_STATE_OBJ_COMMITS:
+  case SR_STATE_OBJ_VALID_AFTER:
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Helper function: This handles the DEL state action using an
+ * <b>obj_type</b> and <b>data</b> needed for the action. */
+static void
+state_query_del_(sr_state_object_t obj_type, void *data)
+{
+  (void) data;
+
+  switch (obj_type) {
+  case SR_STATE_OBJ_PREVSRV:
+    tor_free(sr_state->previous_srv);
+    break;
+  case SR_STATE_OBJ_CURSRV:
+    tor_free(sr_state->current_srv);
+    break;
+  case SR_STATE_OBJ_COMMIT:
+  case SR_STATE_OBJ_COMMITS:
+  case SR_STATE_OBJ_PHASE:
+  case SR_STATE_OBJ_VALID_AFTER:
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Query state using an <b>action</b> for an object type <b>obj_type</b>.
+ * The <b>data</b> pointer needs to point to an object that the action needs
+ * to use and if anything is required to be returned, it is stored in
+ * <b>out</b>.
+ *
+ * This mechanism exists so we have one single point where we synchronized
+ * our memory state with our disk state for every actions that changes it.
+ * We then trigger a write on disk immediately.
+ *
+ * This should be the only entry point to our memory state. It's used by all
+ * our state accessors and should be in the future. */
+static void
+state_query(sr_state_action_t action, sr_state_object_t obj_type,
+            void *data, void **out)
+{
+  switch (action) {
+  case SR_STATE_ACTION_GET:
+    *out = state_query_get_(obj_type, data);
+    break;
+  case SR_STATE_ACTION_PUT:
+    state_query_put_(obj_type, data);
+    break;
+  case SR_STATE_ACTION_DEL:
+    state_query_del_(obj_type, data);
+    break;
+  case SR_STATE_ACTION_DEL_ALL:
+    state_query_del_all_(obj_type);
+    break;
+  case SR_STATE_ACTION_SAVE:
+    /* Only trigger a disk state save. */
+    break;
+  default:
+    tor_assert(0);
+  }
+
+  /* If the action actually changes the state, immediately save it to disk.
+   * The following will sync the state -> disk state and then save it. */
+  if (action != SR_STATE_ACTION_GET) {
+    disk_state_save_to_disk();
+  }
+}
+
+/* Delete the current SRV value from the state freeing it and the value is set
+ * to NULL meaning empty. */
+static void
+state_del_current_srv(void)
+{
+  state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_CURSRV, NULL, NULL);
+}
+
+/* Delete the previous SRV value from the state freeing it and the value is
+ * set to NULL meaning empty. */
+static void
+state_del_previous_srv(void)
+{
+  state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_PREVSRV, NULL, NULL);
+}
+
+/* Rotate SRV value by freeing the previous value, assigning the current
+ * value to the previous one and nullifying the current one. */
+STATIC void
+state_rotate_srv(void)
+{
+  /* First delete previous SRV from the state. Object will be freed. */
+  state_del_previous_srv();
+  /* Set previous SRV with the current one. */
+  sr_state_set_previous_srv(sr_state_get_current_srv());
+  /* Nullify the current srv. */
+  sr_state_set_current_srv(NULL);
+}
+
+/* Set valid after time in the our state. */
+void
+sr_state_set_valid_after(time_t valid_after)
+{
+  state_query(SR_STATE_ACTION_PUT, SR_STATE_OBJ_VALID_AFTER,
+              (void *) &valid_after, NULL);
+}
+
+/* Return the phase we are currently in according to our state. */
+sr_phase_t
+sr_state_get_phase(void)
+{
+  void *ptr;
+  state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_PHASE, NULL, &ptr);
+  return *(sr_phase_t *) ptr;
+}
+
+/* Return the previous SRV value from our state. Value CAN be NULL. */
+const sr_srv_t *
+sr_state_get_previous_srv(void)
+{
+  const sr_srv_t *srv;
+  state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_PREVSRV, NULL,
+              (void *) &srv);
+  return srv;
+}
+
+/* Set the current SRV value from our state. Value CAN be NULL. The srv
+ * object ownership is transferred to the state object. */
+void
+sr_state_set_previous_srv(const sr_srv_t *srv)
+{
+  state_query(SR_STATE_ACTION_PUT, SR_STATE_OBJ_PREVSRV, (void *) srv,
+              NULL);
+}
+
+/* Return the current SRV value from our state. Value CAN be NULL. */
+const sr_srv_t *
+sr_state_get_current_srv(void)
+{
+  const sr_srv_t *srv;
+  state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_CURSRV, NULL,
+              (void *) &srv);
+  return srv;
+}
+
+/* Set the current SRV value from our state. Value CAN be NULL. The srv
+ * object ownership is transferred to the state object. */
+void
+sr_state_set_current_srv(const sr_srv_t *srv)
+{
+  state_query(SR_STATE_ACTION_PUT, SR_STATE_OBJ_CURSRV, (void *) srv,
+              NULL);
+}
+
+/* Clean all the SRVs in our state. */
+void
+sr_state_clean_srvs(void)
+{
+  /* Remove SRVs from state. They will be set to NULL as "empty". */
+  state_del_previous_srv();
+  state_del_current_srv();
+}
+
+/* Return a pointer to the commits map from our state. CANNOT be NULL. */
+digestmap_t *
+sr_state_get_commits(void)
+{
+  digestmap_t *commits;
+  state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_COMMITS,
+              NULL, (void *) &commits);
+  tor_assert(commits);
+  return commits;
+}
+
+/* Update the current SR state as needed for the upcoming voting round at
+ * <b>valid_after</b>. */
+void
+sr_state_update(time_t valid_after)
+{
+  sr_phase_t next_phase;
+
+  tor_assert(sr_state);
+
+  /* Don't call this function twice in the same voting period. */
+  if (valid_after <= sr_state->valid_after) {
+    log_info(LD_DIR, "SR: Asked to update state twice. Ignoring.");
+    return;
+  }
+
+  /* Get phase of upcoming round. */
+  next_phase = get_sr_protocol_phase(valid_after);
+
+  /* If we are transitioning to a new protocol phase, prepare the stage. */
+  if (is_phase_transition(next_phase)) {
+    if (next_phase == SR_PHASE_COMMIT) {
+      /* Going into commit phase means we are starting a new protocol run. */
+      new_protocol_run(valid_after);
+    }
+    /* Set the new phase for this round */
+    sr_state->phase = next_phase;
+  } else if (sr_state->phase == SR_PHASE_COMMIT &&
+             digestmap_size(sr_state->commits) == 0) {
+    /* We are _NOT_ in a transition phase so if we are in the commit phase
+     * and have no commit, generate one. Chances are that we are booting up
+     * so let's have a commit in our state for the next voting period. */
+    sr_commit_t *our_commit =
+      sr_generate_our_commit(valid_after, get_my_v3_authority_cert());
+    if (our_commit) {
+      /* Add our commitment to our state. In case we are unable to create one
+       * (highly unlikely), we won't vote for this protocol run since our
+       * commitment won't be in our state. */
+      sr_state_add_commit(our_commit);
+    }
+  }
+
+  sr_state_set_valid_after(valid_after);
+
+  /* Count the current round */
+  if (sr_state->phase == SR_PHASE_COMMIT) {
+    /* invariant check: we've not entered reveal phase yet */
+    tor_assert(sr_state->n_reveal_rounds == 0);
+    sr_state->n_commit_rounds++;
+  } else {
+    sr_state->n_reveal_rounds++;
+  }
+
+  { /* Debugging. */
+    char tbuf[ISO_TIME_LEN + 1];
+    format_iso_time(tbuf, valid_after);
+    log_info(LD_DIR, "SR: State prepared for upcoming voting period (%s). "
+             "Upcoming phase is %s (counters: %d commit & %d reveal rounds).",
+             tbuf, get_phase_str(sr_state->phase),
+             sr_state->n_commit_rounds, sr_state->n_reveal_rounds);
+  }
+}
+
+/* Return commit object from the given authority digest <b>rsa_identity</b>.
+ * Return NULL if not found. */
+sr_commit_t *
+sr_state_get_commit(const char *rsa_identity)
+{
+  sr_commit_t *commit;
+
+  tor_assert(rsa_identity);
+
+  state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_COMMIT,
+              (void *) rsa_identity, (void *) &commit);
+  return commit;
+}
+
+/* Add <b>commit</b> to the permanent state. The commit object ownership is
+ * transferred to the state so the caller MUST not free it. */
+void
+sr_state_add_commit(sr_commit_t *commit)
+{
+  tor_assert(commit);
+
+  /* Put the commit to the global state. */
+  state_query(SR_STATE_ACTION_PUT, SR_STATE_OBJ_COMMIT,
+              (void *) commit, NULL);
+
+  log_debug(LD_DIR, "SR: Commit from %s has been added to our state.",
+            sr_commit_get_rsa_fpr(commit));
+}
+
+/* Remove all commits from our state. */
+void
+sr_state_delete_commits(void)
+{
+  state_query(SR_STATE_ACTION_DEL_ALL, SR_STATE_OBJ_COMMIT, NULL, NULL);
+}
+
+/* Copy the reveal information from <b>commit</b> into <b>saved_commit</b>.
+ * This <b>saved_commit</b> MUST come from our current SR state. Once modified,
+ * the disk state is updated. */
+void
+sr_state_copy_reveal_info(sr_commit_t *saved_commit, const sr_commit_t *commit)
+{
+  tor_assert(saved_commit);
+  tor_assert(commit);
+
+  saved_commit->reveal_ts = commit->reveal_ts;
+  memcpy(saved_commit->random_number, commit->random_number,
+         sizeof(saved_commit->random_number));
+
+  strlcpy(saved_commit->encoded_reveal, commit->encoded_reveal,
+          sizeof(saved_commit->encoded_reveal));
+  state_query(SR_STATE_ACTION_SAVE, 0, NULL, NULL);
+  log_debug(LD_DIR, "SR: Reveal value learned %s (for commit %s) from %s",
+            saved_commit->encoded_reveal, saved_commit->encoded_commit,
+            sr_commit_get_rsa_fpr(saved_commit));
+}
+
+/* Set the fresh SRV flag from our state. This doesn't need to trigger a
+ * disk state synchronization so we directly change the state. */
+void
+sr_state_set_fresh_srv(void)
+{
+  sr_state->is_srv_fresh = 1;
+}
+
+/* Unset the fresh SRV flag from our state. This doesn't need to trigger a
+ * disk state synchronization so we directly change the state. */
+void
+sr_state_unset_fresh_srv(void)
+{
+  sr_state->is_srv_fresh = 0;
+}
+
+/* Return the value of the fresh SRV flag. */
+unsigned int
+sr_state_srv_is_fresh(void)
+{
+  return sr_state->is_srv_fresh;
+}
+
+/* Cleanup and free our disk and memory state. */
+void
+sr_state_free_all(void)
+{
+  state_free(sr_state);
+  disk_state_free(sr_disk_state);
+  /* Nullify our global state. */
+  sr_state = NULL;
+  sr_disk_state = NULL;
+}
+
+/* Save our current state in memory to disk. */
+void
+sr_state_save(void)
+{
+  /* Query a SAVE action on our current state so it's synced and saved. */
+  state_query(SR_STATE_ACTION_SAVE, 0, NULL, NULL);
+}
+
+/* Return 1 iff the state has been initialized that is it exists in memory.
+ * Return 0 otherwise. */
+int
+sr_state_is_initialized(void)
+{
+  return sr_state == NULL ? 0 : 1;
+}
+
+/* Initialize the disk and memory state.
+ *
+ * If save_to_disk is set to 1, the state is immediately saved to disk after
+ * creation else it's not thus only kept in memory.
+ * If read_from_disk is set to 1, we try to load the state from the disk and
+ * if not found, a new state is created.
+ *
+ * Return 0 on success else a negative value on error. */
+int
+sr_state_init(int save_to_disk, int read_from_disk)
+{
+  int ret = -ENOENT;
+  time_t now = time(NULL);
+
+  /* We shouldn't have those assigned. */
+  tor_assert(sr_disk_state == NULL);
+  tor_assert(sr_state == NULL);
+
+  /* First, try to load the state from disk. */
+  if (read_from_disk) {
+    ret = disk_state_load_from_disk();
+  }
+
+  if (ret < 0) {
+    switch (-ret) {
+    case EINVAL:
+      /* We have a state on disk but it contains something we couldn't parse
+       * or an invalid entry in the state file. Let's remove it since it's
+       * obviously unusable and replace it by an new fresh state below. */
+    case ENOENT:
+      {
+        /* No state on disk so allocate our states for the first time. */
+        sr_state_t *new_state = state_new(default_fname, now);
+        sr_disk_state_t *new_disk_state = disk_state_new(now);
+        state_set(new_state);
+        /* It's important to set our disk state pointer since the save call
+         * below uses it to synchronized it with our memory state.  */
+        disk_state_set(new_disk_state);
+        /* No entry, let's save our new state to disk. */
+        if (save_to_disk && disk_state_save_to_disk() < 0) {
+          goto error;
+        }
+        break;
+      }
+    default:
+      /* Big problem. Not possible. */
+      tor_assert(0);
+    }
+  }
+  /* We have a state in memory, let's make sure it's updated for the current
+   * and next voting round. */
+  {
+    time_t valid_after = voting_schedule_get_next_valid_after_time();
+    sr_state_update(valid_after);
+  }
+  return 0;
+
+ error:
+  return -1;
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/* Set the current phase of the protocol. Used only by unit tests. */
+void
+set_sr_phase(sr_phase_t phase)
+{
+  tor_assert(sr_state);
+  sr_state->phase = phase;
+}
+
+/* Get the SR state. Used only by unit tests */
+sr_state_t *
+get_sr_state(void)
+{
+  return sr_state;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/dirauth/shared_random_state.h b/src/feature/dirauth/shared_random_state.h
new file mode 100644
index 0000000000..f99874872b
--- /dev/null
+++ b/src/feature/dirauth/shared_random_state.h
@@ -0,0 +1,146 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_SHARED_RANDOM_STATE_H
+#define TOR_SHARED_RANDOM_STATE_H
+
+#include "or/dirauth/shared_random.h"
+
+/* Action that can be performed on the state for any objects. */
+typedef enum {
+  SR_STATE_ACTION_GET     = 1,
+  SR_STATE_ACTION_PUT     = 2,
+  SR_STATE_ACTION_DEL     = 3,
+  SR_STATE_ACTION_DEL_ALL = 4,
+  SR_STATE_ACTION_SAVE    = 5,
+} sr_state_action_t;
+
+/* Object in the state that can be queried through the state API. */
+typedef enum {
+  /* Will return a single commit using an authority identity key. */
+  SR_STATE_OBJ_COMMIT,
+  /* Returns the entire list of commits from the state. */
+  SR_STATE_OBJ_COMMITS,
+  /* Return the current SRV object pointer. */
+  SR_STATE_OBJ_CURSRV,
+  /* Return the previous SRV object pointer. */
+  SR_STATE_OBJ_PREVSRV,
+  /* Return the phase. */
+  SR_STATE_OBJ_PHASE,
+  /* Get or Put the valid after time. */
+  SR_STATE_OBJ_VALID_AFTER,
+} sr_state_object_t;
+
+/* State of the protocol. It's also saved on disk in fname. This data
+ * structure MUST be synchronized at all time with the one on disk. */
+typedef struct sr_state_t {
+  /* Filename of the state file on disk. */
+  char *fname;
+  /* Version of the protocol. */
+  uint32_t version;
+  /* The valid-after of the voting period we have prepared the state for. */
+  time_t valid_after;
+  /* Until when is this state valid? */
+  time_t valid_until;
+  /* Protocol phase. */
+  sr_phase_t phase;
+
+  /* Number of runs completed. */
+  uint64_t n_protocol_runs;
+  /* The number of commitment rounds we've performed in this protocol run. */
+  unsigned int n_commit_rounds;
+  /* The number of reveal rounds we've performed in this protocol run. */
+  unsigned int n_reveal_rounds;
+
+  /* A map of all the received commitments for this protocol run. This is
+   * indexed by authority RSA identity digest. */
+  digestmap_t *commits;
+
+  /* Current and previous shared random value. */
+  sr_srv_t *previous_srv;
+  sr_srv_t *current_srv;
+
+  /* Indicate if the state contains an SRV that was _just_ generated. This is
+   * used during voting so that we know whether to use the super majority rule
+   * or not when deciding on keeping it for the consensus. It is _always_ set
+   * to 0 post consensus.
+   *
+   * EDGE CASE: if an authority computes a new SRV then immediately reboots
+   * and, once back up, votes for the current round, it won't know if the
+   * SRV is fresh or not ultimately making it _NOT_ use the super majority
+   * when deciding to put or not the SRV in the consensus. This is for now
+   * an acceptable very rare edge case. */
+  unsigned int is_srv_fresh:1;
+} sr_state_t;
+
+/* Persistent state of the protocol, as saved to disk. */
+typedef struct sr_disk_state_t {
+  uint32_t magic_;
+  /* Version of the protocol. */
+  int Version;
+  /* Version of our running tor. */
+  char *TorVersion;
+  /* Creation time of this state */
+  time_t ValidAfter;
+  /* State valid until? */
+  time_t ValidUntil;
+  /* All commits seen that are valid. */
+  struct config_line_t *Commit;
+  /* Previous and current shared random value. */
+  struct config_line_t *SharedRandValues;
+  /* Extra Lines for configuration we might not know. */
+  struct config_line_t *ExtraLines;
+} sr_disk_state_t;
+
+/* API */
+
+/* Public methods: */
+
+void sr_state_update(time_t valid_after);
+
+/* Private methods (only used by shared-random.c): */
+
+void sr_state_set_valid_after(time_t valid_after);
+sr_phase_t sr_state_get_phase(void);
+const sr_srv_t *sr_state_get_previous_srv(void);
+const sr_srv_t *sr_state_get_current_srv(void);
+void sr_state_set_previous_srv(const sr_srv_t *srv);
+void sr_state_set_current_srv(const sr_srv_t *srv);
+void sr_state_clean_srvs(void);
+digestmap_t *sr_state_get_commits(void);
+sr_commit_t *sr_state_get_commit(const char *rsa_fpr);
+void sr_state_add_commit(sr_commit_t *commit);
+void sr_state_delete_commits(void);
+void sr_state_copy_reveal_info(sr_commit_t *saved_commit,
+                               const sr_commit_t *commit);
+unsigned int sr_state_srv_is_fresh(void);
+void sr_state_set_fresh_srv(void);
+void sr_state_unset_fresh_srv(void);
+int sr_state_init(int save_to_disk, int read_from_disk);
+int sr_state_is_initialized(void);
+void sr_state_save(void);
+void sr_state_free_all(void);
+
+#ifdef SHARED_RANDOM_STATE_PRIVATE
+
+STATIC int disk_state_load_from_disk_impl(const char *fname);
+
+STATIC sr_phase_t get_sr_protocol_phase(time_t valid_after);
+
+STATIC time_t get_state_valid_until_time(time_t now);
+STATIC const char *get_phase_str(sr_phase_t phase);
+STATIC void reset_state_for_new_protocol_run(time_t valid_after);
+STATIC void new_protocol_run(time_t valid_after);
+STATIC void state_rotate_srv(void);
+STATIC int is_phase_transition(sr_phase_t next_phase);
+
+#endif /* defined(SHARED_RANDOM_STATE_PRIVATE) */
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC void set_sr_phase(sr_phase_t phase);
+STATIC sr_state_t *get_sr_state(void);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_SHARED_RANDOM_STATE_H) */
diff --git a/src/feature/dirauth/vote_microdesc_hash_st.h b/src/feature/dirauth/vote_microdesc_hash_st.h
new file mode 100644
index 0000000000..31fc98040e
--- /dev/null
+++ b/src/feature/dirauth/vote_microdesc_hash_st.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef VOTE_MICRODESC_HASH_ST_H
+#define VOTE_MICRODESC_HASH_ST_H
+
+/** Linked list of microdesc hash lines for a single router in a directory
+ * vote.
+ */
+struct vote_microdesc_hash_t {
+  /** Next element in the list, or NULL. */
+  struct vote_microdesc_hash_t *next;
+  /** The raw contents of the microdesc hash line, from the "m" through the
+   * newline. */
+  char *microdesc_hash_line;
+};
+
+#endif
+
diff --git a/src/feature/dircache/cached_dir_st.h b/src/feature/dircache/cached_dir_st.h
new file mode 100644
index 0000000000..38ae86d975
--- /dev/null
+++ b/src/feature/dircache/cached_dir_st.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CACHED_DIR_ST_H
+#define CACHED_DIR_ST_H
+
+/** A cached_dir_t represents a cacheable directory object, along with its
+ * compressed form. */
+struct cached_dir_t {
+  char *dir; /**< Contents of this object, NUL-terminated. */
+  char *dir_compressed; /**< Compressed contents of this object. */
+  size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */
+  size_t dir_compressed_len; /**< Length of <b>dir_compressed</b>. */
+  time_t published; /**< When was this object published. */
+  common_digests_t digests; /**< Digests of this object (networkstatus only) */
+  /** Sha3 digest (also ns only) */
+  uint8_t digest_sha3_as_signed[DIGEST256_LEN];
+  int refcnt; /**< Reference count for this cached_dir_t. */
+};
+
+#endif
+
diff --git a/src/feature/dircache/conscache.c b/src/feature/dircache/conscache.c
new file mode 100644
index 0000000000..bc5928ff23
--- /dev/null
+++ b/src/feature/dircache/conscache.c
@@ -0,0 +1,627 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or/or.h"
+
+#include "or/config.h"
+#include "or/conscache.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/fs/storagedir.h"
+#include "lib/encoding/confline.h"
+
+#define CCE_MAGIC 0x17162253
+
+#ifdef _WIN32
+/* On Windows, unlink won't work on a file if the file is actively mmap()ed.
+ * That forces us to be less aggressive about unlinking files, and causes other
+ * changes throughout our logic.
+ */
+#define MUST_UNMAP_TO_UNLINK
+#endif /* defined(_WIN32) */
+
+/**
+ * A consensus_cache_entry_t is a reference-counted handle to an
+ * item in a consensus_cache_t.  It can be mmapped into RAM, or not,
+ * depending whether it's currently in use.
+ */
+struct consensus_cache_entry_t {
+  uint32_t magic; /**< Must be set to CCE_MAGIC */
+  HANDLE_ENTRY(consensus_cache_entry, consensus_cache_entry_t);
+  int32_t refcnt; /**< Reference count. */
+  unsigned can_remove : 1; /**< If true, we want to delete this file. */
+  /** If true, we intend to unmap this file as soon as we're done with it. */
+  unsigned release_aggressively : 1;
+
+  /** Filename for this object within the storage_dir_t */
+  char *fname;
+  /** Labels associated with this object. Immutable once the object
+   * is created. */
+  config_line_t *labels;
+  /** Pointer to the cache that includes this entry (if any). */
+  consensus_cache_t *in_cache;
+
+  /** Since what time has this object been mapped into RAM, but with the cache
+   * being the only having a reference to it? */
+  time_t unused_since;
+  /** mmaped contents of the underlying file.  May be NULL */
+  tor_mmap_t *map;
+  /** Length of the body within <b>map</b>. */
+  size_t bodylen;
+  /** Pointer to the body within <b>map</b>. */
+  const uint8_t *body;
+};
+
+/**
+ * A consensus_cache_t holds a directory full of labeled items.
+ */
+struct consensus_cache_t {
+  /** Underling storage_dir_t to handle persistence */
+  storage_dir_t *dir;
+  /** List of all the entries in the directory. */
+  smartlist_t *entries;
+
+  /** The maximum number of entries that we'd like to allow in this cache.
+   * This is the same as the storagedir limit when MUST_UNMAP_TO_UNLINK is
+   * not defined. */
+  unsigned max_entries;
+};
+
+static void consensus_cache_clear(consensus_cache_t *cache);
+static void consensus_cache_rescan(consensus_cache_t *);
+static void consensus_cache_entry_map(consensus_cache_t *,
+                                      consensus_cache_entry_t *);
+static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);
+
+/**
+ * Helper: Open a consensus cache in subdirectory <b>subdir</b> of the
+ * data directory, to hold up to <b>max_entries</b> of data.
+ */
+consensus_cache_t *
+consensus_cache_open(const char *subdir, int max_entries)
+{
+  int storagedir_max_entries;
+  consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
+  char *directory = get_cachedir_fname(subdir);
+  cache->max_entries = max_entries;
+
+#ifdef MUST_UNMAP_TO_UNLINK
+  /* If we can't unlink the files that we're still using, then we need to
+   * tell the storagedir backend to allow far more files than this consensus
+   * cache actually wants, so that it can hold files which, from this cache's
+   * perspective, have become useless.
+   */
+#define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000)
+  storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT;
+#else /* !(defined(MUST_UNMAP_TO_UNLINK)) */
+  /* Otherwise, we can just tell the storagedir to use the same limits
+   * as this cache. */
+  storagedir_max_entries = max_entries;
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+
+  cache->dir = storage_dir_new(directory, storagedir_max_entries);
+  tor_free(directory);
+  if (!cache->dir) {
+    tor_free(cache);
+    return NULL;
+  }
+
+  consensus_cache_rescan(cache);
+  return cache;
+}
+
+/** Return true if it's okay to put more entries in this cache than
+ * its official file limit.
+ *
+ * (We need this method on Windows, where we can't unlink files that are still
+ * in use, and therefore might need to temporarily exceed the file limit until
+ * the no-longer-wanted files are deletable.)
+ */
+int
+consensus_cache_may_overallocate(consensus_cache_t *cache)
+{
+  (void) cache;
+#ifdef MUST_UNMAP_TO_UNLINK
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that <b>cache</b> will need.
+ */
+int
+consensus_cache_register_with_sandbox(consensus_cache_t *cache,
+                                      struct sandbox_cfg_elem **cfg)
+{
+#ifdef MUST_UNMAP_TO_UNLINK
+  /* Our Linux sandbox doesn't support huge file lists like the one that would
+   * be generated by using VERY_LARGE_STORAGEDIR_LIMIT above in
+   * consensus_cache_open().  Since the Linux sandbox is the only one we have
+   * right now, we just assert that we never reach this point when we've had
+   * to use VERY_LARGE_STORAGEDIR_LIMIT.
+   *
+   * If at some point in the future we have a different sandbox mechanism that
+   * can handle huge file lists, we can remove this assertion or make it
+   * conditional.
+   */
+  tor_assert_nonfatal_unreached();
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+  return storage_dir_register_with_sandbox(cache->dir, cfg);
+}
+
+/**
+ * Helper: clear all entries from <b>cache</b> (but do not delete
+ * any that aren't marked for removal
+ */
+static void
+consensus_cache_clear(consensus_cache_t *cache)
+{
+  consensus_cache_delete_pending(cache, 0);
+
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    ent->in_cache = NULL;
+    consensus_cache_entry_decref(ent);
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(cache->entries);
+  cache->entries = NULL;
+}
+
+/**
+ * Drop all storage held by <b>cache</b>.
+ */
+void
+consensus_cache_free_(consensus_cache_t *cache)
+{
+  if (! cache)
+    return;
+
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+  storage_dir_free(cache->dir);
+  tor_free(cache);
+}
+
+/**
+ * Write <b>datalen</b> bytes of data at <b>data</b> into the <b>cache</b>,
+ * labeling that data with <b>labels</b>.  On failure, return NULL. On
+ * success, return a newly created consensus_cache_entry_t.
+ *
+ * The returned value will be owned by the cache, and you will have a
+ * reference to it.  Call consensus_cache_entry_decref() when you are
+ * done with it.
+ *
+ * The provided <b>labels</b> MUST have distinct keys: if they don't,
+ * this API does not specify which values (if any) for the duplicate keys
+ * will be considered.
+ */
+consensus_cache_entry_t *
+consensus_cache_add(consensus_cache_t *cache,
+                    const config_line_t *labels,
+                    const uint8_t *data,
+                    size_t datalen)
+{
+  char *fname = NULL;
+  int r = storage_dir_save_labeled_to_file(cache->dir,
+                                            labels, data, datalen, &fname);
+  if (r < 0 || fname == NULL) {
+    return NULL;
+  }
+  consensus_cache_entry_t *ent =
+    tor_malloc_zero(sizeof(consensus_cache_entry_t));
+  ent->magic = CCE_MAGIC;
+  ent->fname = fname;
+  ent->labels = config_lines_dup(labels);
+  ent->in_cache = cache;
+  ent->unused_since = TIME_MAX;
+  smartlist_add(cache->entries, ent);
+  /* Start the reference count at 2: the caller owns one copy, and the
+   * cache owns another.
+   */
+  ent->refcnt = 2;
+
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, return some entry for which <b>key</b>=<b>value</b>.
+ * Return NULL if no such entry exists.
+ *
+ * Does not adjust reference counts.
+ */
+consensus_cache_entry_t *
+consensus_cache_find_first(consensus_cache_t *cache,
+                           const char *key,
+                           const char *value)
+{
+  smartlist_t *tmp = smartlist_new();
+  consensus_cache_find_all(tmp, cache, key, value);
+  consensus_cache_entry_t *ent = NULL;
+  if (smartlist_len(tmp))
+    ent = smartlist_get(tmp, 0);
+  smartlist_free(tmp);
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, add every entry to <b>out<b> for which
+ * <b>key</b>=<b>value</b>.  If <b>key</b> is NULL, add every entry.
+ *
+ * Do not add any entry that has been marked for removal.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_find_all(smartlist_t *out,
+                         consensus_cache_t *cache,
+                         const char *key,
+                         const char *value)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    if (ent->can_remove == 1) {
+      /* We want to delete this; pretend it isn't there. */
+      continue;
+    }
+    if (! key) {
+      smartlist_add(out, ent);
+      continue;
+    }
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (found_val && !strcmp(value, found_val)) {
+      smartlist_add(out, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Given a list of consensus_cache_entry_t, remove all those entries
+ * that do not have <b>key</b>=<b>value</b> in their labels.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_filter_list(smartlist_t *lst,
+                            const char *key,
+                            const char *value)
+{
+  if (BUG(lst == NULL))
+    return; // LCOV_EXCL_LINE
+  if (key == NULL)
+    return;
+  SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) {
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (! found_val || strcmp(value, found_val)) {
+      SMARTLIST_DEL_CURRENT(lst, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * If <b>ent</b> has a label with the given <b>key</b>, return its
+ * value.  Otherwise return NULL.
+ *
+ * The return value is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const char *
+consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                const char *key)
+{
+  const config_line_t *match = config_line_find(ent->labels, key);
+  if (match)
+    return match->value;
+  else
+    return NULL;
+}
+
+/**
+ * Return a pointer to the labels in <b>ent</b>.
+ *
+ * This pointer is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const config_line_t *
+consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
+{
+  return ent->labels;
+}
+
+/**
+ * Increase the reference count of <b>ent</b>.
+ */
+void
+consensus_cache_entry_incref(consensus_cache_entry_t *ent)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+  ++ent->refcnt;
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Release a reference held to <b>ent</b>.
+ *
+ * If it was the last reference, ent will be freed. Therefore, you must not
+ * use <b>ent</b> after calling this function.
+ */
+void
+consensus_cache_entry_decref(consensus_cache_entry_t *ent)
+{
+  if (! ent)
+    return;
+  if (BUG(ent->refcnt <= 0))
+    return; // LCOV_EXCL_LINE
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+
+  --ent->refcnt;
+
+  if (ent->refcnt == 1 && ent->in_cache) {
+    /* Only the cache has a reference: we don't need to keep the file
+     * mapped */
+    if (ent->map) {
+      if (ent->release_aggressively) {
+        consensus_cache_entry_unmap(ent);
+      } else {
+        ent->unused_since = approx_time();
+      }
+    }
+    return;
+  }
+
+  if (ent->refcnt > 0)
+    return;
+
+  /* Refcount is zero; we can free it. */
+  if (ent->map) {
+    consensus_cache_entry_unmap(ent);
+  }
+  tor_free(ent->fname);
+  config_free_lines(ent->labels);
+  consensus_cache_entry_handles_clear(ent);
+  memwipe(ent, 0, sizeof(consensus_cache_entry_t));
+  tor_free(ent);
+}
+
+/**
+ * Mark <b>ent</b> for deletion from the cache.  Deletion will not occur
+ * until the cache is the only place that holds a reference to <b>ent</b>.
+ */
+void
+consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
+{
+  ent->can_remove = 1;
+}
+
+/**
+ * Mark <b>ent</b> as the kind of entry that we don't need to keep mmap'd for
+ * any longer than we're actually using it.
+ */
+void
+consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
+{
+  ent->release_aggressively = 1;
+}
+
+/**
+ * Try to read the body of <b>ent</b> into memory if it isn't already
+ * loaded.  On success, set *<b>body_out</b> to the body, *<b>sz_out</b>
+ * to its size, and return 0.  On failure return -1.
+ *
+ * The resulting body pointer will only be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+int
+consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                               const uint8_t **body_out,
+                               size_t *sz_out)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return -1; // LCOV_EXCL_LINE
+
+  if (! ent->map) {
+    if (! ent->in_cache)
+      return -1;
+
+    consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
+                              (consensus_cache_entry_t *)ent);
+    if (! ent->map) {
+      return -1;
+    }
+  }
+
+  *body_out = ent->body;
+  *sz_out = ent->bodylen;
+  return 0;
+}
+
+/**
+ * Unmap every mmap'd element of <b>cache</b> that has been unused
+ * since <b>cutoff</b>.
+ */
+void
+consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+      /* Somebody is using this entry right now */
+      continue;
+    }
+    if (ent->unused_since > cutoff) {
+      /* Has been unused only for a little while */
+      continue;
+    }
+    if (ent->map == NULL) {
+      /* Not actually mapped. */
+      continue;
+    }
+    consensus_cache_entry_unmap(ent);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Return the number of currently unused filenames available in this cache.
+ */
+int
+consensus_cache_get_n_filenames_available(consensus_cache_t *cache)
+{
+  tor_assert(cache);
+  int max = cache->max_entries;
+  int used = smartlist_len(storage_dir_list(cache->dir));
+#ifdef MUST_UNMAP_TO_UNLINK
+  if (used > max)
+    return 0;
+#else
+  tor_assert_nonfatal(max >= used);
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+  return max - used;
+}
+
+/**
+ * Delete every element of <b>cache</b> has been marked with
+ * consensus_cache_entry_mark_for_removal. If <b>force</b> is false,
+ * retain those entries which are in use by something other than the cache.
+ */
+void
+consensus_cache_delete_pending(consensus_cache_t *cache, int force)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    int force_ent = force;
+#ifdef MUST_UNMAP_TO_UNLINK
+    /* We cannot delete anything with an active mmap on win32, so no
+     * force-deletion. */
+    if (ent->map) {
+      force_ent = 0;
+    }
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+    if (! force_ent) {
+      if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+        /* Somebody is using this entry right now */
+        continue;
+      }
+    }
+    if (ent->can_remove == 0) {
+      /* Don't want to delete this. */
+      continue;
+    }
+    if (BUG(ent->refcnt <= 0)) {
+      continue; // LCOV_EXCL_LINE
+    }
+
+    SMARTLIST_DEL_CURRENT(cache->entries, ent);
+    ent->in_cache = NULL;
+    char *fname = tor_strdup(ent->fname); /* save a copy */
+    consensus_cache_entry_decref(ent);
+    storage_dir_remove_file(cache->dir, fname);
+    tor_free(fname);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Internal helper: rescan <b>cache</b> and rebuild its list of entries.
+ */
+static void
+consensus_cache_rescan(consensus_cache_t *cache)
+{
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+
+  cache->entries = smartlist_new();
+  const smartlist_t *fnames = storage_dir_list(cache->dir);
+  SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
+    tor_mmap_t *map = NULL;
+    config_line_t *labels = NULL;
+    const uint8_t *body;
+    size_t bodylen;
+    map = storage_dir_map_labeled(cache->dir, fname,
+                                  &labels, &body, &bodylen);
+    if (! map) {
+      /* The ERANGE error might come from tor_mmap_file() -- it means the file
+       * was empty. EINVAL might come from ..map_labeled() -- it means the
+       * file was misformatted. In both cases, we should just delete it.
+       */
+      if (errno == ERANGE || errno == EINVAL) {
+        log_warn(LD_FS, "Found %s file %s in consensus cache; removing it.",
+                 errno == ERANGE ? "empty" : "misformatted",
+                 escaped(fname));
+        storage_dir_remove_file(cache->dir, fname);
+      } else {
+        /* Can't load this; continue */
+        log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
+                 escaped(fname), strerror(errno));
+      }
+      continue;
+    }
+    consensus_cache_entry_t *ent =
+      tor_malloc_zero(sizeof(consensus_cache_entry_t));
+    ent->magic = CCE_MAGIC;
+    ent->fname = tor_strdup(fname);
+    ent->labels = labels;
+    ent->refcnt = 1;
+    ent->in_cache = cache;
+    ent->unused_since = TIME_MAX;
+    smartlist_add(cache->entries, ent);
+    tor_munmap_file(map); /* don't actually need to keep this around */
+  } SMARTLIST_FOREACH_END(fname);
+}
+
+/**
+ * Make sure that <b>ent</b> is mapped into RAM.
+ */
+static void
+consensus_cache_entry_map(consensus_cache_t *cache,
+                          consensus_cache_entry_t *ent)
+{
+  if (ent->map)
+    return;
+
+  ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
+                                     NULL, &ent->body, &ent->bodylen);
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Unmap <b>ent</b> from RAM.
+ *
+ * Do not call this if something other than the cache is holding a reference
+ * to <b>ent</b>
+ */
+static void
+consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
+{
+  ent->unused_since = TIME_MAX;
+  if (!ent->map)
+    return;
+
+  tor_munmap_file(ent->map);
+  ent->map = NULL;
+  ent->body = NULL;
+  ent->bodylen = 0;
+  ent->unused_since = TIME_MAX;
+}
+
+HANDLE_IMPL(consensus_cache_entry, consensus_cache_entry_t, )
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Testing only: Return true iff <b>ent</b> is mapped into memory.
+ *
+ * (In normal operation, this information is not exposed.)
+ */
+int
+consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent)
+{
+  if (ent->map) {
+    tor_assert(ent->body);
+    return 1;
+  } else {
+    tor_assert(!ent->body);
+    return 0;
+  }
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/dircache/conscache.h b/src/feature/dircache/conscache.h
new file mode 100644
index 0000000000..c274a60393
--- /dev/null
+++ b/src/feature/dircache/conscache.h
@@ -0,0 +1,66 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSCACHE_H
+#define TOR_CONSCACHE_H
+
+#include "lib/container/handles.h"
+
+typedef struct consensus_cache_entry_t consensus_cache_entry_t;
+typedef struct consensus_cache_t consensus_cache_t;
+
+HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, )
+#define consensus_cache_entry_handle_free(h)    \
+  FREE_AND_NULL(consensus_cache_entry_handle_t, \
+                consensus_cache_entry_handle_free_, (h))
+
+consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries);
+void consensus_cache_free_(consensus_cache_t *cache);
+#define consensus_cache_free(cache) \
+  FREE_AND_NULL(consensus_cache_t, consensus_cache_free_, (cache))
+struct sandbox_cfg_elem;
+int consensus_cache_may_overallocate(consensus_cache_t *cache);
+int consensus_cache_register_with_sandbox(consensus_cache_t *cache,
+                                          struct sandbox_cfg_elem **cfg);
+void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
+void consensus_cache_delete_pending(consensus_cache_t *cache,
+                                    int force);
+int consensus_cache_get_n_filenames_available(consensus_cache_t *cache);
+consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache,
+                                           const struct config_line_t *labels,
+                                           const uint8_t *data,
+                                           size_t datalen);
+
+consensus_cache_entry_t *consensus_cache_find_first(
+                                             consensus_cache_t *cache,
+                                             const char *key,
+                                             const char *value);
+
+void consensus_cache_find_all(smartlist_t *out,
+                              consensus_cache_t *cache,
+                              const char *key,
+                              const char *value);
+void consensus_cache_filter_list(smartlist_t *lst,
+                                 const char *key,
+                                 const char *value);
+
+const char *consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                            const char *key);
+const struct config_line_t *consensus_cache_entry_get_labels(
+                                          const consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_incref(consensus_cache_entry_t *ent);
+void consensus_cache_entry_decref(consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent);
+void consensus_cache_entry_mark_for_aggressive_release(
+                                            consensus_cache_entry_t *ent);
+int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                                   const uint8_t **body_out,
+                                   size_t *sz_out);
+
+#ifdef TOR_UNIT_TESTS
+int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent);
+#endif
+
+#endif /* !defined(TOR_CONSCACHE_H) */
diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c
new file mode 100644
index 0000000000..6d5183f934
--- /dev/null
+++ b/src/feature/dircache/consdiffmgr.c
@@ -0,0 +1,1945 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file consdiffmsr.c
+ *
+ * \brief consensus diff manager functions
+ *
+ * This module is run by directory authorities and caches in order
+ * to remember a number of past consensus documents, and to generate
+ * and serve the diffs from those documents to the latest consensus.
+ */
+
+#define CONSDIFFMGR_PRIVATE
+
+#include "or/or.h"
+#include "or/config.h"
+#include "or/conscache.h"
+#include "or/consdiff.h"
+#include "or/consdiffmgr.h"
+#include "or/cpuworker.h"
+#include "or/networkstatus.h"
+#include "or/routerparse.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/workqueue.h"
+#include "lib/compress/compress.h"
+#include "lib/encoding/confline.h"
+
+#include "or/networkstatus_st.h"
+#include "or/networkstatus_voter_info_st.h"
+
+/**
+ * Labels to apply to items in the conscache object.
+ *
+ * @{
+ */
+/* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
+#define LABEL_DOCTYPE "document-type"
+/* The valid-after time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_VALID_AFTER "consensus-valid-after"
+/* The fresh-until time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_FRESH_UNTIL "consensus-fresh-until"
+/* The valid-until time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_VALID_UNTIL "consensus-valid-until"
+/* Comma-separated list of hex-encoded identity digests for the voting
+ * authorities. */
+#define LABEL_SIGNATORIES "consensus-signatories"
+/* A hex encoded SHA3 digest of the object, as compressed (if any) */
+#define LABEL_SHA3_DIGEST "sha3-digest"
+/* A hex encoded SHA3 digest of the object before compression. */
+#define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
+/* A hex encoded SHA3 digest-as-signed of a consensus */
+#define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
+/* The flavor of the consensus or consensuses diff */
+#define LABEL_FLAVOR "consensus-flavor"
+/* Diff only: the SHA3 digest-as-signed of the source consensus. */
+#define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
+/* Diff only: the SHA3 digest-in-full of the target consensus. */
+#define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
+/* Diff only: the valid-after date of the source consensus. */
+#define LABEL_FROM_VALID_AFTER "from-valid-after"
+/* What kind of compression was used? */
+#define LABEL_COMPRESSION_TYPE "compression"
+/** @} */
+
+#define DOCTYPE_CONSENSUS "consensus"
+#define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
+
+/**
+ * Underlying directory that stores consensuses and consensus diffs.  Don't
+ * use this directly: use cdm_cache_get() instead.
+ */
+static consensus_cache_t *cons_diff_cache = NULL;
+/**
+ * If true, we have learned at least one new consensus since the
+ * consensus cache was last up-to-date.
+ */
+static int cdm_cache_dirty = 0;
+/**
+ * If true, we have scanned the cache to update our hashtable of diffs.
+ */
+static int cdm_cache_loaded = 0;
+
+/**
+ * Possible status values for cdm_diff_t.cdm_diff_status
+ **/
+typedef enum cdm_diff_status_t {
+  CDM_DIFF_PRESENT=1,
+  CDM_DIFF_IN_PROGRESS=2,
+  CDM_DIFF_ERROR=3,
+} cdm_diff_status_t;
+
+/** Which methods do we use for precompressing diffs? */
+static const compress_method_t compress_diffs_with[] = {
+  NO_METHOD,
+  GZIP_METHOD,
+#ifdef HAVE_LZMA
+  LZMA_METHOD,
+#endif
+#ifdef HAVE_ZSTD
+  ZSTD_METHOD,
+#endif
+};
+
+/**
+ * Event for rescanning the cache.
+ */
+static mainloop_event_t *consdiffmgr_rescan_ev = NULL;
+
+static void consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg);
+static void mark_cdm_cache_dirty(void);
+
+/** How many different methods will we try to use for diff compression? */
+STATIC unsigned
+n_diff_compression_methods(void)
+{
+  return ARRAY_LENGTH(compress_diffs_with);
+}
+
+/** Which methods do we use for precompressing consensuses? */
+static const compress_method_t compress_consensus_with[] = {
+  ZLIB_METHOD,
+#ifdef HAVE_LZMA
+  LZMA_METHOD,
+#endif
+#ifdef HAVE_ZSTD
+  ZSTD_METHOD,
+#endif
+};
+
+/** How many different methods will we try to use for diff compression? */
+STATIC unsigned
+n_consensus_compression_methods(void)
+{
+  return ARRAY_LENGTH(compress_consensus_with);
+}
+
+/** For which compression method do we retain old consensuses?  There's no
+ * need to keep all of them, since we won't be serving them.  We'll
+ * go with ZLIB_METHOD because it's pretty fast and everyone has it.
+ */
+#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
+
+/** Handles pointing to the latest consensus entries as compressed and
+ * stored. */
+static consensus_cache_entry_handle_t *
+                  latest_consensus[N_CONSENSUS_FLAVORS]
+                                  [ARRAY_LENGTH(compress_consensus_with)];
+
+/** Hashtable node used to remember the current status of the diff
+ * from a given sha3 digest to the current consensus.  */
+typedef struct cdm_diff_t {
+  HT_ENTRY(cdm_diff_t) node;
+
+  /** Consensus flavor for this diff (part of ht key) */
+  consensus_flavor_t flavor;
+  /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
+   * ht key) */
+  uint8_t from_sha3[DIGEST256_LEN];
+  /** Method by which the diff is compressed. (part of the ht key */
+  compress_method_t compress_method;
+
+  /** One of the CDM_DIFF_* values, depending on whether this diff
+   * is available, in progress, or impossible to compute. */
+  cdm_diff_status_t cdm_diff_status;
+  /** SHA3-256 digest of the consensus that this diff is _to. */
+  uint8_t target_sha3[DIGEST256_LEN];
+
+  /** Handle to the cache entry for this diff, if any.  We use a handle here
+   * to avoid thinking too hard about cache entry lifetime issues. */
+  consensus_cache_entry_handle_t *entry;
+} cdm_diff_t;
+
+/** Hashtable mapping flavor and source consensus digest to status. */
+static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
+
+/**
+ * Configuration for this module
+ */
+static consdiff_cfg_t consdiff_cfg = {
+  // XXXX I'd like to make this number bigger, but it interferes with the
+  // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
+  // XXXX rules.
+  /* .cache_max_num = */ 128
+};
+
+static int consdiffmgr_ensure_space_for_files(int n);
+static int consensus_queue_compression_work(const char *consensus,
+                                            const networkstatus_t *as_parsed);
+static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
+                                          consensus_cache_entry_t *diff_to);
+static void consdiffmgr_set_cache_flags(void);
+
+/* =====
+ * Hashtable setup
+ * ===== */
+
+/** Helper: hash the key of a cdm_diff_t. */
+static unsigned
+cdm_diff_hash(const cdm_diff_t *diff)
+{
+  uint8_t tmp[DIGEST256_LEN + 2];
+  memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
+  tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
+  tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
+  return (unsigned) siphash24g(tmp, sizeof(tmp));
+}
+/** Helper: compare two cdm_diff_t objects for key equality */
+static int
+cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
+{
+  return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
+    diff1->flavor == diff2->flavor &&
+    diff1->compress_method == diff2->compress_method;
+}
+
+HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
+HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
+             0.6, tor_reallocarray, tor_free_)
+
+#define cdm_diff_free(diff) \
+  FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
+
+/** Release all storage held in <b>diff</b>. */
+static void
+cdm_diff_free_(cdm_diff_t *diff)
+{
+  if (!diff)
+    return;
+  consensus_cache_entry_handle_free(diff->entry);
+  tor_free(diff);
+}
+
+/** Create and return a new cdm_diff_t with the given values.  Does not
+ * add it to the hashtable. */
+static cdm_diff_t *
+cdm_diff_new(consensus_flavor_t flav,
+             const uint8_t *from_sha3,
+             const uint8_t *target_sha3,
+             compress_method_t method)
+{
+  cdm_diff_t *ent;
+  ent = tor_malloc_zero(sizeof(cdm_diff_t));
+  ent->flavor = flav;
+  memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
+  memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
+  ent->compress_method = method;
+  return ent;
+}
+
+/**
+ * Examine the diff hashtable to see whether we know anything about computing
+ * a diff of type <b>flav</b> between consensuses with the two provided
+ * SHA3-256 digests.  If a computation is in progress, or if the computation
+ * has already been tried and failed, return 1.  Otherwise, note the
+ * computation as "in progress" so that we don't reattempt it later, and
+ * return 0.
+ */
+static int
+cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav,
+                                   const uint8_t *from_sha3,
+                                   const uint8_t *target_sha3)
+{
+  struct cdm_diff_t search, *ent;
+  unsigned u;
+  int result = 0;
+  for (u = 0; u < n_diff_compression_methods(); ++u) {
+    compress_method_t method = compress_diffs_with[u];
+    memset(&search, 0, sizeof(cdm_diff_t));
+    search.flavor = flav;
+    search.compress_method = method;
+    memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
+    ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+    if (ent) {
+      tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
+      result = 1;
+      continue;
+    }
+    ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
+    ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
+    HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
+  }
+  return result;
+}
+
+/**
+ * Update the status of the diff of type <b>flav</b> between consensuses with
+ * the two provided SHA3-256 digests, so that its status becomes
+ * <b>status</b>, and its value becomes the <b>handle</b>.  If <b>handle</b>
+ * is NULL, then the old handle (if any) is freed, and replaced with NULL.
+ */
+static void
+cdm_diff_ht_set_status(consensus_flavor_t flav,
+                       const uint8_t *from_sha3,
+                       const uint8_t *to_sha3,
+                       compress_method_t method,
+                       int status,
+                       consensus_cache_entry_handle_t *handle)
+{
+  if (handle == NULL) {
+    tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
+  }
+
+  struct cdm_diff_t search, *ent;
+  memset(&search, 0, sizeof(cdm_diff_t));
+  search.flavor = flav;
+  search.compress_method = method,
+  memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
+  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+  if (!ent) {
+    ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
+    ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
+    HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
+  } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
+    // This can happen under certain really pathological conditions
+    // if we decide we don't care about a diff before it is actually
+    // done computing.
+    return;
+  }
+
+  tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
+
+  ent->cdm_diff_status = status;
+  consensus_cache_entry_handle_free(ent->entry);
+  ent->entry = handle;
+}
+
+/**
+ * Helper: Remove from the hash table every present (actually computed) diff
+ * of type <b>flav</b> whose target digest does not match
+ * <b>unless_target_sha3_matches</b>.
+ *
+ * This function is used for the hash table to throw away references to diffs
+ * that do not lead to the most given consensus of a given flavor.
+ */
+static void
+cdm_diff_ht_purge(consensus_flavor_t flav,
+                  const uint8_t *unless_target_sha3_matches)
+{
+  cdm_diff_t **diff, **next;
+  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
+    cdm_diff_t *this = *diff;
+
+    if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
+        flav == (*diff)->flavor) {
+
+      if (BUG((*diff)->entry == NULL) ||
+          consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
+        /* the underlying entry has gone away; drop this. */
+        next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+        cdm_diff_free(this);
+        continue;
+      }
+
+      if (unless_target_sha3_matches &&
+          fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
+                      DIGEST256_LEN)) {
+        /* target hash doesn't match; drop this. */
+        next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+        cdm_diff_free(this);
+        continue;
+      }
+    }
+    next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
+  }
+}
+
+/**
+ * Helper: initialize <b>cons_diff_cache</b>.
+ */
+static void
+cdm_cache_init(void)
+{
+  unsigned n_entries = consdiff_cfg.cache_max_num * 2;
+
+  tor_assert(cons_diff_cache == NULL);
+  cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
+  if (cons_diff_cache == NULL) {
+    // LCOV_EXCL_START
+    log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
+    tor_assert_unreached();
+    // LCOV_EXCL_STOP
+  } else {
+    consdiffmgr_set_cache_flags();
+  }
+  consdiffmgr_rescan_ev =
+    mainloop_event_postloop_new(consdiffmgr_rescan_cb, NULL);
+  mark_cdm_cache_dirty();
+  cdm_cache_loaded = 0;
+}
+
+/**
+ * Helper: return the consensus_cache_t * that backs this manager,
+ * initializing it if needed.
+ */
+STATIC consensus_cache_t *
+cdm_cache_get(void)
+{
+  if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
+    cdm_cache_init();
+  }
+  return cons_diff_cache;
+}
+
+/**
+ * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
+ * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
+ * with <b>label</b> as its label.
+ */
+static void
+cdm_labels_prepend_sha3(config_line_t **labels,
+                        const char *label,
+                        const uint8_t *body,
+                        size_t bodylen)
+{
+  uint8_t sha3_digest[DIGEST256_LEN];
+  char hexdigest[HEX_DIGEST256_LEN+1];
+  crypto_digest256((char *)sha3_digest,
+                   (const char *)body, bodylen, DIGEST_SHA3_256);
+  base16_encode(hexdigest, sizeof(hexdigest),
+                (const char *)sha3_digest, sizeof(sha3_digest));
+
+  config_line_prepend(labels, label, hexdigest);
+}
+
+/** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
+ * given label, set <b>digest_out</b> to that value (decoded), and return 0.
+ *
+ * Return -1 if there is no such label, and -2 if it is badly formatted. */
+STATIC int
+cdm_entry_get_sha3_value(uint8_t *digest_out,
+                         consensus_cache_entry_t *ent,
+                         const char *label)
+{
+  if (ent == NULL)
+    return -1;
+
+  const char *hex = consensus_cache_entry_get_value(ent, label);
+  if (hex == NULL)
+    return -1;
+
+  int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
+  if (n != DIGEST256_LEN)
+    return -2;
+  else
+    return 0;
+}
+
+/**
+ * Helper: look for a consensus with the given <b>flavor</b> and
+ * <b>valid_after</b> time in the cache. Return that consensus if it's
+ * present, or NULL if it's missing.
+ */
+STATIC consensus_cache_entry_t *
+cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
+{
+  char formatted_time[ISO_TIME_LEN+1];
+  format_iso_time_nospace(formatted_time, valid_after);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  /* We'll filter by valid-after time first, since that should
+   * match the fewest documents. */
+  /* We could add an extra hashtable here, but since we only do this scan
+   * when adding a new consensus, it probably doesn't matter much. */
+  smartlist_t *matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_VALID_AFTER, formatted_time);
+  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+
+  consensus_cache_entry_t *result = NULL;
+  if (smartlist_len(matches)) {
+    result = smartlist_get(matches, 0);
+  }
+  smartlist_free(matches);
+
+  return result;
+}
+
+/** Return the maximum age (in seconds) of consensuses that we should consider
+ * storing. The available space in the directory may impose additional limits
+ * on how much we store. */
+static int32_t
+get_max_age_to_cache(void)
+{
+  const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
+  const int32_t MIN_MAX_AGE_TO_CACHE = 0;
+  const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
+  const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff";
+
+  const or_options_t *options = get_options();
+
+  if (options->MaxConsensusAgeForDiffs) {
+    const int v = options->MaxConsensusAgeForDiffs;
+    if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
+      return MAX_MAX_AGE_TO_CACHE;
+    else
+      return v;
+  }
+
+  /* The parameter is in hours, so we multiply */
+  return 3600 * networkstatus_get_param(NULL,
+                                        MAX_AGE_TO_CACHE_NAME,
+                                        DEFAULT_MAX_AGE_TO_CACHE,
+                                        MIN_MAX_AGE_TO_CACHE,
+                                        MAX_MAX_AGE_TO_CACHE);
+}
+
+/**
+ * Given a string containing a networkstatus consensus, and the results of
+ * having parsed that consensus, add that consensus to the cache if it is not
+ * already present and not too old.  Create new consensus diffs from or to
+ * that consensus as appropriate.
+ *
+ * Return 0 on success and -1 on failure.
+ */
+int
+consdiffmgr_add_consensus(const char *consensus,
+                          const networkstatus_t *as_parsed)
+{
+  if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
+    return -1; // LCOV_EXCL_LINE
+  if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
+    return -1; // LCOV_EXCL_LINE
+
+  const consensus_flavor_t flavor = as_parsed->flavor;
+  const time_t valid_after = as_parsed->valid_after;
+
+  if (valid_after < approx_time() - get_max_age_to_cache()) {
+    log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
+             "too old.");
+    return -1;
+  }
+
+  /* Do we already have this one? */
+  consensus_cache_entry_t *entry =
+    cdm_cache_lookup_consensus(flavor, valid_after);
+  if (entry) {
+    log_info(LD_DIRSERV, "We already have a copy of that consensus");
+    return -1;
+  }
+
+  /* We don't have it. Add it to the cache. */
+  return consensus_queue_compression_work(consensus, as_parsed);
+}
+
+/**
+ * Helper: used to sort two smartlists of consensus_cache_entry_t by their
+ * LABEL_VALID_AFTER labels.
+ */
+static int
+compare_by_valid_after_(const void **a, const void **b)
+{
+  const consensus_cache_entry_t *e1 = *a;
+  const consensus_cache_entry_t *e2 = *b;
+  /* We're in luck here: sorting UTC iso-encoded values lexically will work
+   * fine (until 9999). */
+  return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
+                    consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
+}
+
+/**
+ * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
+ * entry.
+ */
+static consensus_cache_entry_t *
+sort_and_find_most_recent(smartlist_t *lst)
+{
+  smartlist_sort(lst, compare_by_valid_after_);
+  if (smartlist_len(lst)) {
+    return smartlist_get(lst, smartlist_len(lst) - 1);
+  } else {
+    return NULL;
+  }
+}
+
+/** Return i such that compress_consensus_with[i] == method. Return
+ * -1 if no such i exists. */
+static int
+consensus_compression_method_pos(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < n_consensus_compression_methods(); ++i) {
+    if (compress_consensus_with[i] == method) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+/**
+ * If we know a consensus with the flavor <b>flavor</b> compressed with
+ * <b>method</b>, set *<b>entry_out</b> to that value.  Return values are as
+ * for consdiffmgr_find_diff_from().
+ */
+consdiff_status_t
+consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           compress_method_t method)
+{
+  tor_assert(entry_out);
+  tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
+
+  int pos = consensus_compression_method_pos(method);
+  if (pos < 0) {
+     // We don't compress consensuses with this method.
+    return CONSDIFF_NOT_FOUND;
+  }
+  consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
+  if (!handle)
+    return CONSDIFF_NOT_FOUND;
+  *entry_out = consensus_cache_entry_handle_get(handle);
+  if (*entry_out)
+    return CONSDIFF_AVAILABLE;
+  else
+    return CONSDIFF_NOT_FOUND;
+}
+
+/**
+ * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
+ * from the source consensus with the specified digest (which must be SHA3).
+ *
+ * If the diff is present, store it into *<b>entry_out</b> and return
+ * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
+ * CONSDIFF_IN_PROGRESS.
+ */
+consdiff_status_t
+consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           int digest_type,
+                           const uint8_t *digest,
+                           size_t digestlen,
+                           compress_method_t method)
+{
+  if (BUG(digest_type != DIGEST_SHA3_256) ||
+      BUG(digestlen != DIGEST256_LEN)) {
+    return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
+  }
+
+  // Try to look up the entry in the hashtable.
+  cdm_diff_t search, *ent;
+  memset(&search, 0, sizeof(search));
+  search.flavor = flavor;
+  search.compress_method = method;
+  memcpy(search.from_sha3, digest, DIGEST256_LEN);
+  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+
+  if (ent == NULL ||
+      ent->cdm_diff_status == CDM_DIFF_ERROR) {
+    return CONSDIFF_NOT_FOUND;
+  } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
+    return CONSDIFF_IN_PROGRESS;
+  } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
+    return CONSDIFF_IN_PROGRESS;
+  }
+
+  if (BUG(ent->entry == NULL)) {
+    return CONSDIFF_NOT_FOUND;
+  }
+  *entry_out = consensus_cache_entry_handle_get(ent->entry);
+  return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
+
+#if 0
+  // XXXX Remove this.  I'm keeping it around for now in case we need to
+  // XXXX debug issues in the hashtable.
+  char hex[HEX_DIGEST256_LEN+1];
+  base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  smartlist_t *matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_FROM_SHA3_DIGEST, hex);
+  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+
+  *entry_out = sort_and_find_most_recent(matches);
+  consdiff_status_t result =
+    (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
+  smartlist_free(matches);
+
+  return result;
+#endif /* 0 */
+}
+
+/**
+ * Perform periodic cleanup tasks on the consensus diff cache.  Return
+ * the number of objects marked for deletion.
+ */
+int
+consdiffmgr_cleanup(void)
+{
+  smartlist_t *objects = smartlist_new();
+  smartlist_t *consensuses = smartlist_new();
+  smartlist_t *diffs = smartlist_new();
+  int n_to_delete = 0;
+
+  log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
+
+  // 1. Delete any consensus or diff or anything whose valid_after is too old.
+  const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache();
+
+  consensus_cache_find_all(objects, cdm_cache_get(),
+                           NULL, NULL);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    const char *lv_valid_after =
+      consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+    if (! lv_valid_after) {
+      log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
+                LABEL_VALID_AFTER);
+      continue;
+    }
+    time_t valid_after = 0;
+    if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
+      log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
+                "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
+      continue;
+    }
+    if (valid_after < valid_after_cutoff) {
+      log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
+                "too old", LABEL_VALID_AFTER, lv_valid_after);
+      consensus_cache_entry_mark_for_removal(ent);
+      ++n_to_delete;
+    }
+  } SMARTLIST_FOREACH_END(ent);
+
+  // 2. Delete all diffs that lead to a consensus whose valid-after is not the
+  // latest.
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    /* Determine the most recent consensus of this flavor */
+    consensus_cache_find_all(consensuses, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
+    consensus_cache_entry_t *most_recent =
+      sort_and_find_most_recent(consensuses);
+    if (most_recent == NULL)
+      continue;
+    const char *most_recent_sha3 =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    if (BUG(most_recent_sha3 == NULL))
+      continue; // LCOV_EXCL_LINE
+
+    /* consider all such-flavored diffs, and look to see if they match. */
+    consensus_cache_find_all(diffs, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+    consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
+    SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+      const char *this_diff_target_sha3 =
+        consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
+      if (!this_diff_target_sha3)
+        continue;
+      if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
+        consensus_cache_entry_mark_for_removal(diff);
+        ++n_to_delete;
+      }
+    } SMARTLIST_FOREACH_END(diff);
+    smartlist_clear(consensuses);
+    smartlist_clear(diffs);
+  }
+
+  // 3. Delete all consensuses except the most recent that are compressed with
+  // an un-preferred method.
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    /* Determine the most recent consensus of this flavor */
+    consensus_cache_find_all(consensuses, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
+    consensus_cache_entry_t *most_recent =
+      sort_and_find_most_recent(consensuses);
+    if (most_recent == NULL)
+      continue;
+    const char *most_recent_sha3_uncompressed =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    const char *retain_methodname = compression_method_get_name(
+                               RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
+
+    if (BUG(most_recent_sha3_uncompressed == NULL))
+      continue;
+    SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
+      const char *lv_sha3_uncompressed =
+        consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
+      if (BUG(! lv_sha3_uncompressed))
+        continue;
+      if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
+        continue; // This _is_ the most recent.
+      const char *lv_methodname =
+        consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+      if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
+        consensus_cache_entry_mark_for_removal(ent);
+        ++n_to_delete;
+      }
+    } SMARTLIST_FOREACH_END(ent);
+  }
+
+  smartlist_free(objects);
+  smartlist_free(consensuses);
+  smartlist_free(diffs);
+
+  // Actually remove files, if they're not used.
+  consensus_cache_delete_pending(cdm_cache_get(), 0);
+  return n_to_delete;
+}
+
+/**
+ * Initialize the consensus diff manager and its cache, and configure
+ * its parameters based on the latest torrc and networkstatus parameters.
+ */
+void
+consdiffmgr_configure(const consdiff_cfg_t *cfg)
+{
+  if (cfg)
+    memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
+
+  (void) cdm_cache_get();
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that the consensus diff manager will need.
+ */
+int
+consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
+{
+  return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
+}
+
+/**
+ * Scan the consensus diff manager's cache for any grossly malformed entries,
+ * and mark them as deletable.  Return 0 if no problems were found; 1
+ * if problems were found and fixed.
+ */
+int
+consdiffmgr_validate(void)
+{
+  /* Right now, we only check for entries that have bad sha3 values */
+  int problems = 0;
+
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cdm_cache_get(),
+                           NULL, NULL);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) {
+    uint8_t sha3_expected[DIGEST256_LEN];
+    uint8_t sha3_received[DIGEST256_LEN];
+    int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
+    if (r == -1) {
+      /* digest isn't there; that's allowed */
+      continue;
+    } else if (r == -2) {
+      /* digest is malformed; that's not allowed */
+      problems = 1;
+      consensus_cache_entry_mark_for_removal(obj);
+      continue;
+    }
+    const uint8_t *body;
+    size_t bodylen;
+    consensus_cache_entry_incref(obj);
+    r = consensus_cache_entry_get_body(obj, &body, &bodylen);
+    if (r == 0) {
+      crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
+                       DIGEST_SHA3_256);
+    }
+    consensus_cache_entry_decref(obj);
+    if (r < 0)
+      continue;
+
+    // Deconfuse coverity about the possibility of sha3_received being
+    // uninitialized
+    tor_assert(r <= 0);
+
+    if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
+      problems = 1;
+      consensus_cache_entry_mark_for_removal(obj);
+      continue;
+    }
+
+  } SMARTLIST_FOREACH_END(obj);
+  smartlist_free(objects);
+  return problems;
+}
+
+/**
+ * Helper: build new diffs of <b>flavor</b> as needed
+ */
+static void
+consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
+{
+  smartlist_t *matches = NULL;
+  smartlist_t *diffs = NULL;
+  smartlist_t *compute_diffs_from = NULL;
+  strmap_t *have_diff_from = NULL;
+
+  // look for the most recent consensus, and for all previous in-range
+  // consensuses.  Do they all have diffs to it?
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  // 1. find the most recent consensus, and the ones that we might want
+  //    to diff to it.
+  const char *methodname = compression_method_get_name(
+                             RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
+
+  matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+  consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
+  consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
+  if (!most_recent) {
+    log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
+             "not making diffs", flavname);
+    goto done;
+  }
+  tor_assert(smartlist_len(matches));
+  smartlist_del(matches, smartlist_len(matches) - 1);
+
+  const char *most_recent_valid_after =
+    consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
+  if (BUG(most_recent_valid_after == NULL))
+    goto done; //LCOV_EXCL_LINE
+  uint8_t most_recent_sha3[DIGEST256_LEN];
+  if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
+                                   LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
+    goto done; //LCOV_EXCL_LINE
+
+  // 2. Find all the relevant diffs _to_ this consensus. These are ones
+  //    that we don't need to compute.
+  diffs = smartlist_new();
+  consensus_cache_find_all(diffs, cdm_cache_get(),
+                           LABEL_VALID_AFTER, most_recent_valid_after);
+  consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+  consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
+  have_diff_from = strmap_new();
+  SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+    const char *va = consensus_cache_entry_get_value(diff,
+                                                     LABEL_FROM_VALID_AFTER);
+    if (BUG(va == NULL))
+      continue; // LCOV_EXCL_LINE
+    strmap_set(have_diff_from, va, diff);
+  } SMARTLIST_FOREACH_END(diff);
+
+  // 3. See which consensuses in 'matches' don't have diffs yet.
+  smartlist_reverse(matches); // from newest to oldest.
+  compute_diffs_from = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
+    const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+    if (BUG(va == NULL))
+      continue; // LCOV_EXCL_LINE
+    if (strmap_get(have_diff_from, va) != NULL)
+      continue; /* we already have this one. */
+    smartlist_add(compute_diffs_from, ent);
+    /* Since we are not going to serve this as the most recent consensus
+     * any more, we should stop keeping it mmap'd when it's not in use.
+     */
+    consensus_cache_entry_mark_for_aggressive_release(ent);
+  } SMARTLIST_FOREACH_END(ent);
+
+  log_info(LD_DIRSERV,
+           "The most recent %s consensus is valid-after %s. We have diffs to "
+           "this consensus for %d/%d older %s consensuses. Generating diffs "
+           "for the other %d.",
+           flavname,
+           most_recent_valid_after,
+           smartlist_len(matches) - smartlist_len(compute_diffs_from),
+           smartlist_len(matches),
+           flavname,
+           smartlist_len(compute_diffs_from));
+
+  // 4. Update the hashtable; remove entries in this flavor to other
+  //    target consensuses.
+  cdm_diff_ht_purge(flavor, most_recent_sha3);
+
+  // 5. Actually launch the requests.
+  SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
+    if (BUG(c == most_recent))
+      continue; // LCOV_EXCL_LINE
+
+    uint8_t this_sha3[DIGEST256_LEN];
+    if (cdm_entry_get_sha3_value(this_sha3, c,
+                                 LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
+      // Not actually a bug, since we might be running with a directory
+      // with stale files from before the #22143 fixes.
+      continue;
+    }
+    if (cdm_diff_ht_check_and_note_pending(flavor,
+                                           this_sha3, most_recent_sha3)) {
+      // This is already pending, or we encountered an error.
+      continue;
+    }
+    consensus_diff_queue_diff_work(c, most_recent);
+  } SMARTLIST_FOREACH_END(c);
+
+ done:
+  smartlist_free(matches);
+  smartlist_free(diffs);
+  smartlist_free(compute_diffs_from);
+  strmap_free(have_diff_from, NULL);
+}
+
+/**
+ * Scan the cache for the latest consensuses and add their handles to
+ * latest_consensus
+ */
+static void
+consdiffmgr_consensus_load(void)
+{
+  smartlist_t *matches = smartlist_new();
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    smartlist_clear(matches);
+    consensus_cache_find_all(matches, cdm_cache_get(),
+                             LABEL_FLAVOR, flavname);
+    consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
+    if (! most_recent)
+      continue; // no consensuses.
+    const char *most_recent_sha3 =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    if (BUG(most_recent_sha3 == NULL))
+      continue; // LCOV_EXCL_LINE
+    consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                                most_recent_sha3);
+
+    // Everything that remains matches the most recent consensus of this
+    // flavor.
+    SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
+      const char *lv_compression =
+        consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+      compress_method_t method =
+        compression_method_get_by_name(lv_compression);
+      int pos = consensus_compression_method_pos(method);
+      if (pos < 0)
+        continue;
+      consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
+      latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
+    } SMARTLIST_FOREACH_END(ent);
+  }
+  smartlist_free(matches);
+}
+
+/**
+ * Scan the cache for diffs, and add them to the hashtable.
+ */
+static void
+consdiffmgr_diffs_load(void)
+{
+  smartlist_t *diffs = smartlist_new();
+  consensus_cache_find_all(diffs, cdm_cache_get(),
+                           LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+  SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+    const char *lv_flavor =
+      consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
+    if (!lv_flavor)
+      continue;
+    int flavor = networkstatus_parse_flavor_name(lv_flavor);
+    if (flavor < 0)
+      continue;
+    const char *lv_compression =
+      consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
+    compress_method_t method = NO_METHOD;
+    if (lv_compression) {
+      method = compression_method_get_by_name(lv_compression);
+      if (method == UNKNOWN_METHOD) {
+        continue;
+      }
+    }
+
+    uint8_t from_sha3[DIGEST256_LEN];
+    uint8_t to_sha3[DIGEST256_LEN];
+    if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
+      continue;
+    if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
+      continue;
+
+    cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
+                           method,
+                           CDM_DIFF_PRESENT,
+                           consensus_cache_entry_handle_new(diff));
+  } SMARTLIST_FOREACH_END(diff);
+  smartlist_free(diffs);
+}
+
+/**
+ * Build new diffs as needed.
+ */
+void
+consdiffmgr_rescan(void)
+{
+  if (cdm_cache_dirty == 0)
+    return;
+
+  // Clean up here to make room for new diffs, and to ensure that older
+  // consensuses do not have any entries.
+  consdiffmgr_cleanup();
+
+  if (cdm_cache_loaded == 0) {
+    consdiffmgr_diffs_load();
+    consdiffmgr_consensus_load();
+    cdm_cache_loaded = 1;
+  }
+
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    consdiffmgr_rescan_flavor_((consensus_flavor_t) flav);
+  }
+
+  cdm_cache_dirty = 0;
+}
+
+/** Callback wrapper for consdiffmgr_rescan */
+static void
+consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  consdiffmgr_rescan();
+}
+
+/** Mark the cache as dirty, and schedule a rescan event. */
+static void
+mark_cdm_cache_dirty(void)
+{
+  cdm_cache_dirty = 1;
+  tor_assert(consdiffmgr_rescan_ev);
+  mainloop_event_activate(consdiffmgr_rescan_ev);
+}
+
+/**
+ * Helper: compare two files by their from-valid-after and valid-after labels,
+ * trying to sort in ascending order by from-valid-after (when present) and
+ * valid-after (when not).  Place everything that has neither label first in
+ * the list.
+ */
+static int
+compare_by_staleness_(const void **a, const void **b)
+{
+  const consensus_cache_entry_t *e1 = *a;
+  const consensus_cache_entry_t *e2 = *b;
+  const char *va1, *fva1, *va2, *fva2;
+  va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
+  va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
+  fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
+  fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
+
+  if (fva1)
+    va1 = fva1;
+  if (fva2)
+    va2 = fva2;
+
+  /* See note about iso-encoded values in compare_by_valid_after_.  Also note
+   * that missing dates will get placed first. */
+  return strcmp_opt(va1, va2);
+}
+
+/** If there are not enough unused filenames to store <b>n</b> files, then
+ * delete old consensuses until there are.  (We have to keep track of the
+ * number of filenames because of the way that the seccomp2 cache works.)
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+static int
+consdiffmgr_ensure_space_for_files(int n)
+{
+  consensus_cache_t *cache = cdm_cache_get();
+  if (consensus_cache_get_n_filenames_available(cache) >= n) {
+    // there are already enough unused filenames.
+    return 0;
+  }
+  // Try a cheap deletion of stuff that's waiting to get deleted.
+  consensus_cache_delete_pending(cache, 0);
+  if (consensus_cache_get_n_filenames_available(cache) >= n) {
+    // okay, _that_ made enough filenames available.
+    return 0;
+  }
+  // Let's get more assertive: clean out unused stuff, and force-remove
+  // the files that we can.
+  consdiffmgr_cleanup();
+  consensus_cache_delete_pending(cache, 1);
+  const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
+  if (n_to_remove <= 0) {
+    // okay, finally!
+    return 0;
+  }
+
+  // At this point, we're going to have to throw out objects that will be
+  // missed.  Too bad!
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cache, NULL, NULL);
+  smartlist_sort(objects, compare_by_staleness_);
+  int n_marked = 0;
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    consensus_cache_entry_mark_for_removal(ent);
+    if (++n_marked >= n_to_remove)
+      break;
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(objects);
+
+  consensus_cache_delete_pending(cache, 1);
+
+  if (consensus_cache_may_overallocate(cache)) {
+    /* If we're allowed to throw extra files into the cache, let's do so
+     * rather getting upset.
+     */
+    return 0;
+  }
+
+  if (BUG(n_marked < n_to_remove))
+    return -1;
+  else
+    return 0;
+}
+
+/**
+ * Set consensus cache flags on the objects in this consdiffmgr.
+ */
+static void
+consdiffmgr_set_cache_flags(void)
+{
+  /* Right now, we just mark the consensus objects for aggressive release,
+   * so that they get mmapped for as little time as possible. */
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE,
+                           DOCTYPE_CONSENSUS);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    consensus_cache_entry_mark_for_aggressive_release(ent);
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(objects);
+}
+
+/**
+ * Called before shutdown: drop all storage held by the consdiffmgr.c module.
+ */
+void
+consdiffmgr_free_all(void)
+{
+  cdm_diff_t **diff, **next;
+  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
+    cdm_diff_t *this = *diff;
+    next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+    cdm_diff_free(this);
+  }
+  int i;
+  unsigned j;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    for (j = 0; j < n_consensus_compression_methods(); ++j) {
+      consensus_cache_entry_handle_free(latest_consensus[i][j]);
+    }
+  }
+  memset(latest_consensus, 0, sizeof(latest_consensus));
+  consensus_cache_free(cons_diff_cache);
+  cons_diff_cache = NULL;
+  mainloop_event_free(consdiffmgr_rescan_ev);
+}
+
+/* =====
+   Thread workers
+   =====*/
+
+typedef struct compressed_result_t {
+  config_line_t *labels;
+  /**
+   * Output: Body of the diff, as compressed.
+   */
+  uint8_t *body;
+  /**
+   * Output: length of body_out
+   */
+  size_t bodylen;
+} compressed_result_t;
+
+/**
+ * Compress the bytestring <b>input</b> of length <b>len</b> using the
+ * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
+ *
+ * For each successful compression, set the fields in the <b>results_out</b>
+ * array in the position corresponding to the compression method. Use
+ * <b>labels_in</b> as a basis for the labels of the result.
+ *
+ * Return 0 if all compression succeeded; -1 if any failed.
+ */
+static int
+compress_multiple(compressed_result_t *results_out, int n_methods,
+                  const compress_method_t *methods,
+                  const uint8_t *input, size_t len,
+                  const config_line_t *labels_in)
+{
+  int rv = 0;
+  int i;
+  for (i = 0; i < n_methods; ++i) {
+    compress_method_t method = methods[i];
+    const char *methodname = compression_method_get_name(method);
+    char *result;
+    size_t sz;
+    if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
+      results_out[i].body = (uint8_t*)result;
+      results_out[i].bodylen = sz;
+      results_out[i].labels = config_lines_dup(labels_in);
+      cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
+                              results_out[i].body,
+                              results_out[i].bodylen);
+      config_line_prepend(&results_out[i].labels,
+                          LABEL_COMPRESSION_TYPE,
+                          methodname);
+    } else {
+      rv = -1;
+    }
+  }
+  return rv;
+}
+
+/**
+ * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
+ * as produced by compress_multiple, store them all into the
+ * consdiffmgr, and store handles to them in the <b>handles_out</b>
+ * array.
+ *
+ * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
+ * was stored.
+ */
+static cdm_diff_status_t
+store_multiple(consensus_cache_entry_handle_t **handles_out,
+               int n,
+               const compress_method_t *methods,
+               const compressed_result_t *results,
+               const char *description)
+{
+  cdm_diff_status_t status = CDM_DIFF_ERROR;
+  consdiffmgr_ensure_space_for_files(n);
+
+  int i;
+  for (i = 0; i < n; ++i) {
+    compress_method_t method = methods[i];
+    uint8_t *body_out = results[i].body;
+    size_t bodylen_out = results[i].bodylen;
+    config_line_t *labels = results[i].labels;
+    const char *methodname = compression_method_get_name(method);
+    if (body_out && bodylen_out && labels) {
+      /* Success! Store the results */
+      log_info(LD_DIRSERV, "Adding %s, compressed with %s",
+               description, methodname);
+
+      consensus_cache_entry_t *ent =
+        consensus_cache_add(cdm_cache_get(),
+                            labels,
+                            body_out,
+                            bodylen_out);
+      if (ent == NULL) {
+        static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
+        log_fn_ratelim(&cant_store_ratelim, LOG_WARN, LD_FS,
+                       "Unable to store object %s compressed with %s.",
+                       description, methodname);
+        continue;
+      }
+
+      status = CDM_DIFF_PRESENT;
+      handles_out[i] = consensus_cache_entry_handle_new(ent);
+      consensus_cache_entry_decref(ent);
+    }
+  }
+  return status;
+}
+
+/**
+ * An object passed to a worker thread that will try to produce a consensus
+ * diff.
+ */
+typedef struct consensus_diff_worker_job_t {
+  /**
+   * Input: The consensus to compute the diff from.  Holds a reference to the
+   * cache entry, which must not be released until the job is passed back to
+   * the main thread. The body must be mapped into memory in the main thread.
+   */
+  consensus_cache_entry_t *diff_from;
+  /**
+   * Input: The consensus to compute the diff to.  Holds a reference to the
+   * cache entry, which must not be released until the job is passed back to
+   * the main thread. The body must be mapped into memory in the main thread.
+   */
+  consensus_cache_entry_t *diff_to;
+
+  /** Output: labels and bodies */
+  compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)];
+} consensus_diff_worker_job_t;
+
+/** Given a consensus_cache_entry_t, check whether it has a label claiming
+ * that it was compressed.  If so, uncompress its contents into <b>out</b> and
+ * set <b>outlen</b> to hold their size.  If not, just copy the body into
+ * <b>out</b> and set <b>outlen</b> to its length.  Return 0 on success,
+ * -1 on failure.
+ *
+ * In all cases, the output is nul-terminated. */
+STATIC int
+uncompress_or_copy(char **out, size_t *outlen,
+                   consensus_cache_entry_t *ent)
+{
+  const uint8_t *body;
+  size_t bodylen;
+
+  if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
+    return -1;
+
+  const char *lv_compression =
+    consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+  compress_method_t method = NO_METHOD;
+
+  if (lv_compression)
+    method = compression_method_get_by_name(lv_compression);
+
+  return tor_uncompress(out, outlen, (const char *)body, bodylen,
+                        method, 1, LOG_WARN);
+}
+
+/**
+ * Worker function. This function runs inside a worker thread and receives
+ * a consensus_diff_worker_job_t as its input.
+ */
+static workqueue_reply_t
+consensus_diff_worker_threadfn(void *state_, void *work_)
+{
+  (void)state_;
+  consensus_diff_worker_job_t *job = work_;
+  const uint8_t *diff_from, *diff_to;
+  size_t len_from, len_to;
+  int r;
+  /* We need to have the body already mapped into RAM here.
+   */
+  r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
+  if (BUG(r < 0))
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
+  if (BUG(r < 0))
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+
+  const char *lv_to_valid_after =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
+  const char *lv_to_fresh_until =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL);
+  const char *lv_to_valid_until =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL);
+  const char *lv_to_signatories =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES);
+  const char *lv_from_valid_after =
+    consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
+  const char *lv_from_digest =
+    consensus_cache_entry_get_value(job->diff_from,
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
+  const char *lv_from_flavor =
+    consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
+  const char *lv_to_flavor =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
+  const char *lv_to_digest =
+    consensus_cache_entry_get_value(job->diff_to,
+                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+
+  if (! lv_from_digest) {
+    /* This isn't a bug right now, since it can happen if you're migrating
+     * from an older version of master to a newer one.  The older ones didn't
+     * annotate their stored consensus objects with sha3-digest-as-signed.
+    */
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+
+  /* All these values are mandatory on the input */
+  if (BUG(!lv_to_valid_after) ||
+      BUG(!lv_from_valid_after) ||
+      BUG(!lv_from_flavor) ||
+      BUG(!lv_to_flavor)) {
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+  /* The flavors need to match */
+  if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+
+  char *consensus_diff;
+  {
+    char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    size_t diff_from_nt_len, diff_to_nt_len;
+
+    if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
+                           job->diff_from) < 0) {
+      return WQ_RPL_REPLY;
+    }
+    if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
+                           job->diff_to) < 0) {
+      tor_free(diff_from_nt);
+      return WQ_RPL_REPLY;
+    }
+    tor_assert(diff_from_nt);
+    tor_assert(diff_to_nt);
+
+    // XXXX ugh; this is going to calculate the SHA3 of both its
+    // XXXX inputs again, even though we already have that. Maybe it's time
+    // XXXX to change the API here?
+    consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
+    tor_free(diff_from_nt);
+    tor_free(diff_to_nt);
+  }
+  if (!consensus_diff) {
+    /* Couldn't generate consensus; we'll leave the reply blank. */
+    return WQ_RPL_REPLY;
+  }
+
+  /* Compress the results and send the reply */
+  tor_assert(compress_diffs_with[0] == NO_METHOD);
+  size_t difflen = strlen(consensus_diff);
+  job->out[0].body = (uint8_t *) consensus_diff;
+  job->out[0].bodylen = difflen;
+
+  config_line_t *common_labels = NULL;
+  if (lv_to_valid_until)
+    config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until);
+  if (lv_to_fresh_until)
+    config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until);
+  if (lv_to_signatories)
+    config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories);
+  cdm_labels_prepend_sha3(&common_labels,
+                          LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                          job->out[0].body,
+                          job->out[0].bodylen);
+  config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
+                      lv_from_valid_after);
+  config_line_prepend(&common_labels, LABEL_VALID_AFTER,
+                      lv_to_valid_after);
+  config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
+  config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
+                      lv_from_digest);
+  config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
+                      lv_to_digest);
+  config_line_prepend(&common_labels, LABEL_DOCTYPE,
+                      DOCTYPE_CONSENSUS_DIFF);
+
+  job->out[0].labels = config_lines_dup(common_labels);
+  cdm_labels_prepend_sha3(&job->out[0].labels,
+                          LABEL_SHA3_DIGEST,
+                          job->out[0].body,
+                          job->out[0].bodylen);
+
+  compress_multiple(job->out+1,
+                    n_diff_compression_methods()-1,
+                    compress_diffs_with+1,
+                    (const uint8_t*)consensus_diff, difflen, common_labels);
+
+  config_free_lines(common_labels);
+  return WQ_RPL_REPLY;
+}
+
+#define consensus_diff_worker_job_free(job)             \
+  FREE_AND_NULL(consensus_diff_worker_job_t,            \
+                consensus_diff_worker_job_free_, (job))
+
+/**
+ * Helper: release all storage held in <b>job</b>.
+ */
+static void
+consensus_diff_worker_job_free_(consensus_diff_worker_job_t *job)
+{
+  if (!job)
+    return;
+  unsigned u;
+  for (u = 0; u < n_diff_compression_methods(); ++u) {
+    config_free_lines(job->out[u].labels);
+    tor_free(job->out[u].body);
+  }
+  consensus_cache_entry_decref(job->diff_from);
+  consensus_cache_entry_decref(job->diff_to);
+  tor_free(job);
+}
+
+/**
+ * Worker function: This function runs in the main thread, and receives
+ * a consensus_diff_worker_job_t that the worker thread has already
+ * processed.
+ */
+static void
+consensus_diff_worker_replyfn(void *work_)
+{
+  tor_assert(in_main_thread());
+  tor_assert(work_);
+
+  consensus_diff_worker_job_t *job = work_;
+
+  const char *lv_from_digest =
+    consensus_cache_entry_get_value(job->diff_from,
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
+  const char *lv_to_digest =
+    consensus_cache_entry_get_value(job->diff_to,
+                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+  const char *lv_flavor =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
+  if (BUG(lv_from_digest == NULL))
+    lv_from_digest = "???"; // LCOV_EXCL_LINE
+  if (BUG(lv_to_digest == NULL))
+    lv_to_digest = "???"; // LCOV_EXCL_LINE
+
+  uint8_t from_sha3[DIGEST256_LEN];
+  uint8_t to_sha3[DIGEST256_LEN];
+  int flav = -1;
+  int cache = 1;
+  if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
+                                   LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
+    cache = 0;
+  if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
+                                   LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
+    cache = 0;
+  if (BUG(lv_flavor == NULL)) {
+    cache = 0;
+  } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
+    cache = 0;
+  }
+
+  consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
+  memset(handles, 0, sizeof(handles));
+
+  char description[128];
+  tor_snprintf(description, sizeof(description),
+               "consensus diff from %s to %s",
+               lv_from_digest, lv_to_digest);
+
+  int status = store_multiple(handles,
+                              n_diff_compression_methods(),
+                              compress_diffs_with,
+                              job->out,
+                              description);
+
+  if (status != CDM_DIFF_PRESENT) {
+    /* Failure! Nothing to do but complain */
+    log_warn(LD_DIRSERV,
+             "Worker was unable to compute consensus diff "
+             "from %s to %s", lv_from_digest, lv_to_digest);
+    /* Cache this error so we don't try to compute this one again. */
+    status = CDM_DIFF_ERROR;
+  }
+
+  unsigned u;
+  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
+    compress_method_t method = compress_diffs_with[u];
+    if (cache) {
+      consensus_cache_entry_handle_t *h = handles[u];
+      int this_status = status;
+      if (h == NULL) {
+        this_status = CDM_DIFF_ERROR;
+      }
+      tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
+      cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, this_status, h);
+    } else {
+      consensus_cache_entry_handle_free(handles[u]);
+    }
+  }
+
+  consensus_diff_worker_job_free(job);
+}
+
+/**
+ * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
+ * in a worker thread.
+ */
+static int
+consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
+                               consensus_cache_entry_t *diff_to)
+{
+  tor_assert(in_main_thread());
+
+  consensus_cache_entry_incref(diff_from);
+  consensus_cache_entry_incref(diff_to);
+
+  consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
+  job->diff_from = diff_from;
+  job->diff_to = diff_to;
+
+  /* Make sure body is mapped. */
+  const uint8_t *body;
+  size_t bodylen;
+  int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
+  int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
+  if (r1 < 0 || r2 < 0)
+    goto err;
+
+  workqueue_entry_t *work;
+  work = cpuworker_queue_work(WQ_PRI_LOW,
+                              consensus_diff_worker_threadfn,
+                              consensus_diff_worker_replyfn,
+                              job);
+  if (!work)
+    goto err;
+
+  return 0;
+ err:
+  consensus_diff_worker_job_free(job); // includes decrefs.
+  return -1;
+}
+
+/**
+ * Holds requests and replies for consensus_compress_workers.
+ */
+typedef struct consensus_compress_worker_job_t {
+  char *consensus;
+  size_t consensus_len;
+  consensus_flavor_t flavor;
+  config_line_t *labels_in;
+  compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)];
+} consensus_compress_worker_job_t;
+
+#define consensus_compress_worker_job_free(job) \
+  FREE_AND_NULL(consensus_compress_worker_job_t, \
+                consensus_compress_worker_job_free_, (job))
+
+/**
+ * Free all resources held in <b>job</b>
+ */
+static void
+consensus_compress_worker_job_free_(consensus_compress_worker_job_t *job)
+{
+  if (!job)
+    return;
+  tor_free(job->consensus);
+  config_free_lines(job->labels_in);
+  unsigned u;
+  for (u = 0; u < n_consensus_compression_methods(); ++u) {
+    config_free_lines(job->out[u].labels);
+    tor_free(job->out[u].body);
+  }
+  tor_free(job);
+}
+/**
+ * Worker function. This function runs inside a worker thread and receives
+ * a consensus_compress_worker_job_t as its input.
+ */
+static workqueue_reply_t
+consensus_compress_worker_threadfn(void *state_, void *work_)
+{
+  (void)state_;
+  consensus_compress_worker_job_t *job = work_;
+  consensus_flavor_t flavor = job->flavor;
+  const char *consensus = job->consensus;
+  size_t bodylen = job->consensus_len;
+
+  config_line_t *labels = config_lines_dup(job->labels_in);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                          (const uint8_t *)consensus, bodylen);
+  {
+    const char *start, *end;
+    if (router_get_networkstatus_v3_signed_boundaries(consensus,
+                                                        &start, &end) < 0) {
+      start = consensus;
+      end = consensus+bodylen;
+    }
+    cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
+                            (const uint8_t *)start,
+                            end - start);
+  }
+  config_line_prepend(&labels, LABEL_FLAVOR, flavname);
+  config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+
+  compress_multiple(job->out,
+                    n_consensus_compression_methods(),
+                    compress_consensus_with,
+                    (const uint8_t*)consensus, bodylen, labels);
+  config_free_lines(labels);
+  return WQ_RPL_REPLY;
+}
+
+/**
+ * Worker function: This function runs in the main thread, and receives
+ * a consensus_diff_compress_job_t that the worker thread has already
+ * processed.
+ */
+static void
+consensus_compress_worker_replyfn(void *work_)
+{
+  consensus_compress_worker_job_t *job = work_;
+
+  consensus_cache_entry_handle_t *handles[
+                               ARRAY_LENGTH(compress_consensus_with)];
+  memset(handles, 0, sizeof(handles));
+
+  store_multiple(handles,
+                 n_consensus_compression_methods(),
+                 compress_consensus_with,
+                 job->out,
+                 "consensus");
+  mark_cdm_cache_dirty();
+
+  unsigned u;
+  consensus_flavor_t f = job->flavor;
+  tor_assert((int)f < N_CONSENSUS_FLAVORS);
+  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
+    if (handles[u] == NULL)
+      continue;
+    consensus_cache_entry_handle_free(latest_consensus[f][u]);
+    latest_consensus[f][u] = handles[u];
+  }
+
+  consensus_compress_worker_job_free(job);
+}
+
+/**
+ * If true, we compress in worker threads.
+ */
+static int background_compression = 0;
+
+/**
+ * Queue a job to compress <b>consensus</b> and store its compressed
+ * text in the cache.
+ */
+static int
+consensus_queue_compression_work(const char *consensus,
+                                 const networkstatus_t *as_parsed)
+{
+  tor_assert(consensus);
+  tor_assert(as_parsed);
+
+  consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
+  job->consensus = tor_strdup(consensus);
+  job->consensus_len = strlen(consensus);
+  job->flavor = as_parsed->flavor;
+
+  char va_str[ISO_TIME_LEN+1];
+  char vu_str[ISO_TIME_LEN+1];
+  char fu_str[ISO_TIME_LEN+1];
+  format_iso_time_nospace(va_str, as_parsed->valid_after);
+  format_iso_time_nospace(fu_str, as_parsed->fresh_until);
+  format_iso_time_nospace(vu_str, as_parsed->valid_until);
+  config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str);
+  config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str);
+  config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str);
+  if (as_parsed->voters) {
+    smartlist_t *hexvoters = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(as_parsed->voters,
+                            networkstatus_voter_info_t *, vi) {
+      if (smartlist_len(vi->sigs) == 0)
+        continue; // didn't sign.
+      char d[HEX_DIGEST_LEN+1];
+      base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN);
+      smartlist_add_strdup(hexvoters, d);
+    } SMARTLIST_FOREACH_END(vi);
+    char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL);
+    config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers);
+    tor_free(signers);
+    SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp));
+    smartlist_free(hexvoters);
+  }
+
+  if (background_compression) {
+    workqueue_entry_t *work;
+    work = cpuworker_queue_work(WQ_PRI_LOW,
+                                consensus_compress_worker_threadfn,
+                                consensus_compress_worker_replyfn,
+                                job);
+    if (!work) {
+      consensus_compress_worker_job_free(job);
+      return -1;
+    }
+
+    return 0;
+  } else {
+    consensus_compress_worker_threadfn(NULL, job);
+    consensus_compress_worker_replyfn(job);
+    return 0;
+  }
+}
+
+/**
+ * Tell the consdiffmgr backend to compress consensuses in worker threads.
+ */
+void
+consdiffmgr_enable_background_compression(void)
+{
+  // This isn't the default behavior because it would break unit tests.
+  background_compression = 1;
+}
+
+/** Read the set of voters from the cached object <b>ent</b> into
+ * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
+ * -1 if no signatories were recorded. */
+int
+consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent,
+                                           smartlist_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
+  if (s == NULL)
+    return -1;
+  smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
+  return 0;
+}
+
+/** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
+ * and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
+/** Read the valid until timestamp from the cached object <b>ent</b> into
+ * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
+/** Read the valid after timestamp from the cached object <b>ent</b> into
+ * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h
new file mode 100644
index 0000000000..66c3d65002
--- /dev/null
+++ b/src/feature/dircache/consdiffmgr.h
@@ -0,0 +1,75 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSDIFFMGR_H
+#define TOR_CONSDIFFMGR_H
+
+enum compress_method_t;
+
+/**
+ * Possible outcomes from trying to look up a given consensus diff.
+ */
+typedef enum consdiff_status_t {
+  CONSDIFF_AVAILABLE,
+  CONSDIFF_NOT_FOUND,
+  CONSDIFF_IN_PROGRESS,
+} consdiff_status_t;
+
+typedef struct consdiff_cfg_t {
+  int32_t cache_max_num;
+} consdiff_cfg_t;
+
+struct consensus_cache_entry_t; // from conscache.h
+
+int consdiffmgr_add_consensus(const char *consensus,
+                              const networkstatus_t *as_parsed);
+
+consdiff_status_t consdiffmgr_find_consensus(
+                           struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           enum compress_method_t method);
+
+consdiff_status_t consdiffmgr_find_diff_from(
+                           struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           int digest_type,
+                           const uint8_t *digest,
+                           size_t digestlen,
+                           enum compress_method_t method);
+
+int consensus_cache_entry_get_voter_id_digests(
+                                  const struct consensus_cache_entry_t *ent,
+                                  smartlist_t *out);
+int consensus_cache_entry_get_fresh_until(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+int consensus_cache_entry_get_valid_until(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+int consensus_cache_entry_get_valid_after(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+
+void consdiffmgr_rescan(void);
+int consdiffmgr_cleanup(void);
+void consdiffmgr_enable_background_compression(void);
+void consdiffmgr_configure(const consdiff_cfg_t *cfg);
+struct sandbox_cfg_elem;
+int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg);
+void consdiffmgr_free_all(void);
+int consdiffmgr_validate(void);
+
+#ifdef CONSDIFFMGR_PRIVATE
+STATIC unsigned n_diff_compression_methods(void);
+STATIC unsigned n_consensus_compression_methods(void);
+STATIC consensus_cache_t *cdm_cache_get(void);
+STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus(
+                          consensus_flavor_t flavor, time_t valid_after);
+STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out,
+                                    consensus_cache_entry_t *ent,
+                                    const char *label);
+STATIC int uncompress_or_copy(char **out, size_t *outlen,
+                              consensus_cache_entry_t *ent);
+#endif /* defined(CONSDIFFMGR_PRIVATE) */
+
+#endif /* !defined(TOR_CONSDIFFMGR_H) */
diff --git a/src/feature/dircache/directory.c b/src/feature/dircache/directory.c
new file mode 100644
index 0000000000..8f2bd8d3a7
--- /dev/null
+++ b/src/feature/dircache/directory.c
@@ -0,0 +1,5966 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DIRECTORY_PRIVATE
+
+#include "or/or.h"
+#include "lib/err/backtrace.h"
+#include "or/bridges.h"
+#include "lib/container/buffers.h"
+#include "or/circuitbuild.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/connection_edge.h"
+#include "or/conscache.h"
+#include "or/consdiff.h"
+#include "or/consdiffmgr.h"
+#include "or/control.h"
+#include "lib/compress/compress.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/entrynodes.h"
+#include "or/fp_pair.h"
+#include "or/geoip.h"
+#include "or/hs_cache.h"
+#include "or/hs_common.h"
+#include "or/hs_control.h"
+#include "or/hs_client.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/relay.h"
+#include "or/rendclient.h"
+#include "or/rendcommon.h"
+#include "or/rendservice.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+#include "lib/encoding/confline.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+#if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
+#if !defined(OpenBSD)
+#include <malloc.h>
+#endif
+#endif
+
+#include "or/dirauth/dirvote.h"
+#include "or/dirauth/mode.h"
+#include "or/dirauth/shared_random.h"
+
+#include "or/authority_cert_st.h"
+#include "or/cached_dir_st.h"
+#include "or/dir_connection_st.h"
+#include "or/dir_server_st.h"
+#include "or/entry_connection_st.h"
+#include "or/networkstatus_st.h"
+#include "or/node_st.h"
+#include "or/rend_service_descriptor_st.h"
+#include "or/routerinfo_st.h"
+
+/**
+ * \file directory.c
+ * \brief Code to send and fetch information from directory authorities and
+ * caches via HTTP.
+ *
+ * Directory caches and authorities use dirserv.c to generate the results of a
+ * query and stream them to the connection; clients use routerparse.c to parse
+ * them.
+ *
+ * Every directory request has a dir_connection_t on the client side and on
+ * the server side.  In most cases, the dir_connection_t object is a linked
+ * connection, tunneled through an edge_connection_t so that it can be a
+ * stream on the Tor network.  The only non-tunneled connections are those
+ * that are used to upload material (descriptors and votes) to authorities.
+ * Among tunneled connections, some use one-hop circuits, and others use
+ * multi-hop circuits for anonymity.
+ *
+ * Directory requests are launched by calling
+ * directory_initiate_request(). This
+ * launch the connection, will construct an HTTP request with
+ * directory_send_command(), send the and wait for a response.  The client
+ * later handles the response with connection_dir_client_reached_eof(),
+ * which passes the information received to another part of Tor.
+ *
+ * On the server side, requests are read in directory_handle_command(),
+ * which dispatches first on the request type (GET or POST), and then on
+ * the URL requested. GET requests are processed with a table-based
+ * dispatcher in url_table[].  The process of handling larger GET requests
+ * is complicated because we need to avoid allocating a copy of all the
+ * data to be sent to the client in one huge buffer.  Instead, we spool the
+ * data into the buffer using logic in connection_dirserv_flushed_some() in
+ * dirserv.c.  (TODO: If we extended buf.c to have a zero-copy
+ * reference-based buffer type, we could remove most of that code, at the
+ * cost of a bit more reference counting.)
+ **/
+
+/* In-points to directory.c:
+ *
+ * - directory_post_to_dirservers(), called from
+ *   router_upload_dir_desc_to_dirservers() in router.c
+ *   upload_service_descriptor() in rendservice.c
+ * - directory_get_from_dirserver(), called from
+ *   rend_client_refetch_renddesc() in rendclient.c
+ *   run_scheduled_events() in main.c
+ *   do_hup() in main.c
+ * - connection_dir_process_inbuf(), called from
+ *   connection_process_inbuf() in connection.c
+ * - connection_dir_finished_flushing(), called from
+ *   connection_finished_flushing() in connection.c
+ * - connection_dir_finished_connecting(), called from
+ *   connection_finished_connecting() in connection.c
+ */
+static void directory_send_command(dir_connection_t *conn,
+                                   int direct,
+                                   const directory_request_t *request);
+static int body_is_plausible(const char *body, size_t body_len, int purpose);
+static void http_set_address_origin(const char *headers, connection_t *conn);
+static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
+static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn);
+static void connection_dir_download_cert_failed(
+                               dir_connection_t *conn, int status_code);
+static void connection_dir_retry_bridges(smartlist_t *descs);
+static void dir_routerdesc_download_failed(smartlist_t *failed,
+                                           int status_code,
+                                           int router_purpose,
+                                           int was_extrainfo,
+                                           int was_descriptor_digests);
+static void dir_microdesc_download_failed(smartlist_t *failed,
+                                          int status_code,
+                                          const char *dir_id);
+static int client_likes_consensus(const struct consensus_cache_entry_t *ent,
+                                  const char *want_url);
+
+static void connection_dir_close_consensus_fetches(
+                   dir_connection_t *except_this_one, const char *resource);
+
+/********* START VARIABLES **********/
+
+/** Maximum size, in bytes, for resized buffers. */
+#define MAX_BUF_SIZE ((1<<24)-1) /* 16MB-1 */
+/** Maximum size, in bytes, for any directory object that we've downloaded. */
+#define MAX_DIR_DL_SIZE MAX_BUF_SIZE
+
+/** Maximum size, in bytes, for any directory object that we're accepting
+ * as an upload. */
+#define MAX_DIR_UL_SIZE MAX_BUF_SIZE
+
+/** How far in the future do we allow a directory server to tell us it is
+ * before deciding that one of us has the wrong time? */
+#define ALLOW_DIRECTORY_TIME_SKEW (30*60)
+
+#define X_ADDRESS_HEADER "X-Your-Address-Is: "
+#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: "
+
+/** HTTP cache control: how long do we tell proxies they can cache each
+ * kind of document we serve? */
+#define FULL_DIR_CACHE_LIFETIME (60*60)
+#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
+#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
+#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
+#define ROUTERDESC_CACHE_LIFETIME (30*60)
+#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
+#define ROBOTS_CACHE_LIFETIME (24*60*60)
+#define MICRODESC_CACHE_LIFETIME (48*60*60)
+
+/********* END VARIABLES ************/
+
+/** Convert a connection_t* to a dir_connection_t*; assert if the cast is
+ * invalid. */
+dir_connection_t *
+TO_DIR_CONN(connection_t *c)
+{
+  tor_assert(c->magic == DIR_CONNECTION_MAGIC);
+  return DOWNCAST(dir_connection_t, c);
+}
+
+/** Return false if the directory purpose <b>dir_purpose</b>
+ * does not require an anonymous (three-hop) connection.
+ *
+ * Return true 1) by default, 2) if all directory actions have
+ * specifically been configured to be over an anonymous connection,
+ * or 3) if the router is a bridge */
+int
+purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                        const char *resource)
+{
+  if (get_options()->AllDirActionsPrivate)
+    return 1;
+
+  if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
+    if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC
+        && resource && !strcmp(resource, "authority.z")) {
+      /* We are asking a bridge for its own descriptor. That doesn't need
+         anonymity. */
+      return 0;
+    }
+    /* Assume all other bridge stuff needs anonymity. */
+    return 1; /* if no circuits yet, this might break bootstrapping, but it's
+               * needed to be safe. */
+  }
+
+  switch (dir_purpose)
+  {
+    case DIR_PURPOSE_UPLOAD_DIR:
+    case DIR_PURPOSE_UPLOAD_VOTE:
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      return 0;
+    case DIR_PURPOSE_HAS_FETCHED_HSDESC:
+    case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2:
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+    case DIR_PURPOSE_FETCH_HSDESC:
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      return 1;
+    case DIR_PURPOSE_SERVER:
+    default:
+      log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d",
+               dir_purpose, router_purpose);
+      tor_assert_nonfatal_unreached();
+      return 1; /* Assume it needs anonymity; better safe than sorry. */
+  }
+}
+
+/** Return a newly allocated string describing <b>auth</b>. Only describes
+ * authority features. */
+STATIC char *
+authdir_type_to_string(dirinfo_type_t auth)
+{
+  char *result;
+  smartlist_t *lst = smartlist_new();
+  if (auth & V3_DIRINFO)
+    smartlist_add(lst, (void*)"V3");
+  if (auth & BRIDGE_DIRINFO)
+    smartlist_add(lst, (void*)"Bridge");
+  if (smartlist_len(lst)) {
+    result = smartlist_join_strings(lst, ", ", 0, NULL);
+  } else {
+    result = tor_strdup("[Not an authority]");
+  }
+  smartlist_free(lst);
+  return result;
+}
+
+/** Return a string describing a given directory connection purpose. */
+STATIC const char *
+dir_conn_purpose_to_string(int purpose)
+{
+  switch (purpose)
+    {
+    case DIR_PURPOSE_UPLOAD_DIR:
+      return "server descriptor upload";
+    case DIR_PURPOSE_UPLOAD_VOTE:
+      return "server vote upload";
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+      return "consensus signature upload";
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+      return "server descriptor fetch";
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      return "extra-info fetch";
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      return "consensus network-status fetch";
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      return "authority cert fetch";
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+      return "status vote fetch";
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+      return "consensus signature fetch";
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+      return "hidden-service v2 descriptor fetch";
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+      return "hidden-service v2 descriptor upload";
+    case DIR_PURPOSE_FETCH_HSDESC:
+      return "hidden-service descriptor fetch";
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      return "hidden-service descriptor upload";
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      return "microdescriptor fetch";
+    }
+
+  log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
+  return "(unknown)";
+}
+
+/** Return the requisite directory information types. */
+STATIC dirinfo_type_t
+dir_fetch_type(int dir_purpose, int router_purpose, const char *resource)
+{
+  dirinfo_type_t type;
+  switch (dir_purpose) {
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      type = EXTRAINFO_DIRINFO;
+      if (router_purpose == ROUTER_PURPOSE_BRIDGE)
+        type |= BRIDGE_DIRINFO;
+      else
+        type |= V3_DIRINFO;
+      break;
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+      if (router_purpose == ROUTER_PURPOSE_BRIDGE)
+        type = BRIDGE_DIRINFO;
+      else
+        type = V3_DIRINFO;
+      break;
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      type = V3_DIRINFO;
+      break;
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      type = V3_DIRINFO;
+      if (resource && !strcmp(resource, "microdesc"))
+        type |= MICRODESC_DIRINFO;
+      break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      type = MICRODESC_DIRINFO;
+      break;
+    default:
+      log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
+      type = NO_DIRINFO;
+      break;
+  }
+  return type;
+}
+
+/** Return true iff <b>identity_digest</b> is the digest of a router which
+ * says that it caches extrainfos.  (If <b>is_authority</b> we always
+ * believe that to be true.) */
+int
+router_supports_extrainfo(const char *identity_digest, int is_authority)
+{
+  const node_t *node = node_get_by_id(identity_digest);
+
+  if (node && node->ri) {
+    if (node->ri->caches_extra_info)
+      return 1;
+  }
+  if (is_authority) {
+    return 1;
+  }
+  return 0;
+}
+
+/** Return true iff any trusted directory authority has accepted our
+ * server descriptor.
+ *
+ * We consider any authority sufficient because waiting for all of
+ * them means it never happens while any authority is down; we don't
+ * go for something more complex in the middle (like \>1/3 or \>1/2 or
+ * \>=1/2) because that doesn't seem necessary yet.
+ */
+int
+directories_have_accepted_server_descriptor(void)
+{
+  const smartlist_t *servers = router_get_trusted_dir_servers();
+  const or_options_t *options = get_options();
+  SMARTLIST_FOREACH(servers, dir_server_t *, d, {
+    if ((d->type & options->PublishServerDescriptor_) &&
+        d->has_accepted_serverdesc) {
+      return 1;
+    }
+  });
+  return 0;
+}
+
+/** Start a connection to every suitable directory authority, using
+ * connection purpose <b>dir_purpose</b> and uploading <b>payload</b>
+ * (of length <b>payload_len</b>). The dir_purpose should be one of
+ * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'.
+ *
+ * <b>router_purpose</b> describes the type of descriptor we're
+ * publishing, if we're publishing a descriptor -- e.g. general or bridge.
+ *
+ * <b>type</b> specifies what sort of dir authorities (V3,
+ * BRIDGE, etc) we should upload to.
+ *
+ * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
+ * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
+ * bytes of <b>payload</b> hold an extra-info document.  Upload the descriptor
+ * to all authorities, and the extra-info document to all authorities that
+ * support it.
+ */
+void
+directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
+                             dirinfo_type_t type,
+                             const char *payload,
+                             size_t payload_len, size_t extrainfo_len)
+{
+  const or_options_t *options = get_options();
+  dir_indirection_t indirection;
+  const smartlist_t *dirservers = router_get_trusted_dir_servers();
+  int found = 0;
+  const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
+                            dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
+  tor_assert(dirservers);
+  /* This tries dirservers which we believe to be down, but ultimately, that's
+   * harmless, and we may as well err on the side of getting things uploaded.
+   */
+  SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) {
+      routerstatus_t *rs = &(ds->fake_status);
+      size_t upload_len = payload_len;
+
+      if ((type & ds->type) == 0)
+        continue;
+
+      if (exclude_self && router_digest_is_me(ds->digest)) {
+        /* we don't upload to ourselves, but at least there's now at least
+         * one authority of this type that has what we wanted to upload. */
+        found = 1;
+        continue;
+      }
+
+      if (options->StrictNodes &&
+          routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) {
+        log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but "
+                 "it's in our ExcludedNodes list and StrictNodes is set. "
+                 "Skipping.",
+                 ds->nickname,
+                 dir_conn_purpose_to_string(dir_purpose));
+        continue;
+      }
+
+      found = 1; /* at least one authority of this type was listed */
+      if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR)
+        ds->has_accepted_serverdesc = 0;
+
+      if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
+        upload_len += extrainfo_len;
+        log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
+                 (int) extrainfo_len);
+      }
+      if (purpose_needs_anonymity(dir_purpose, router_purpose, NULL)) {
+        indirection = DIRIND_ANONYMOUS;
+      } else if (!fascist_firewall_allows_dir_server(ds,
+                                                     FIREWALL_DIR_CONNECTION,
+                                                     0)) {
+        if (fascist_firewall_allows_dir_server(ds, FIREWALL_OR_CONNECTION, 0))
+          indirection = DIRIND_ONEHOP;
+        else
+          indirection = DIRIND_ANONYMOUS;
+      } else {
+        indirection = DIRIND_DIRECT_CONN;
+      }
+
+      directory_request_t *req = directory_request_new(dir_purpose);
+      directory_request_set_routerstatus(req, rs);
+      directory_request_set_router_purpose(req, router_purpose);
+      directory_request_set_indirection(req, indirection);
+      directory_request_set_payload(req, payload, upload_len);
+      directory_initiate_request(req);
+      directory_request_free(req);
+  } SMARTLIST_FOREACH_END(ds);
+  if (!found) {
+    char *s = authdir_type_to_string(type);
+    log_warn(LD_DIR, "Publishing server descriptor to directory authorities "
+             "of type '%s', but no authorities of that type listed!", s);
+    tor_free(s);
+  }
+}
+
+/** Return true iff, according to the values in <b>options</b>, we should be
+ * using directory guards for direct downloads of directory information. */
+STATIC int
+should_use_directory_guards(const or_options_t *options)
+{
+  /* Public (non-bridge) servers never use directory guards. */
+  if (public_server_mode(options))
+    return 0;
+  /* If guards are disabled, we can't use directory guards.
+   */
+  if (!options->UseEntryGuards)
+    return 0;
+  /* If we're configured to fetch directory info aggressively or of a
+   * nonstandard type, don't use directory guards. */
+  if (options->DownloadExtraInfo || options->FetchDirInfoEarly ||
+      options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors)
+    return 0;
+  return 1;
+}
+
+/** Pick an unconstrained directory server from among our guards, the latest
+ * networkstatus, or the fallback dirservers, for use in downloading
+ * information of type <b>type</b>, and return its routerstatus. */
+static const routerstatus_t *
+directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
+                                 uint8_t dir_purpose,
+                                 circuit_guard_state_t **guard_state_out)
+{
+  const routerstatus_t *rs = NULL;
+  const or_options_t *options = get_options();
+
+  if (options->UseBridges)
+    log_warn(LD_BUG, "Called when we have UseBridges set.");
+
+  if (should_use_directory_guards(options)) {
+    const node_t *node = guards_choose_dirguard(dir_purpose, guard_state_out);
+    if (node)
+      rs = node->rs;
+  } else {
+    /* anybody with a non-zero dirport will do */
+    rs = router_pick_directory_server(type, pds_flags);
+  }
+  if (!rs) {
+    log_info(LD_DIR, "No router found for %s; falling back to "
+             "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
+    rs = router_pick_fallback_dirserver(type, pds_flags);
+  }
+
+  return rs;
+}
+
+/**
+ * Set the extra fields in <b>req</b> that are used when requesting a
+ * consensus of type <b>resource</b>.
+ *
+ * Right now, these fields are if-modified-since and x-or-diff-from-consensus.
+ */
+static void
+dir_consensus_request_set_additional_headers(directory_request_t *req,
+                                             const char *resource)
+{
+  time_t if_modified_since = 0;
+  uint8_t or_diff_from[DIGEST256_LEN];
+  int or_diff_from_is_set = 0;
+
+  /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
+   * period of 1 hour.
+   */
+  const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180;
+  const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72;
+  const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0;
+  const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192;
+  const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] =
+    "try-diff-for-consensus-newer-than";
+
+  int flav = FLAV_NS;
+  if (resource)
+    flav = networkstatus_parse_flavor_name(resource);
+
+  int32_t max_age_for_diff = 3600 *
+    networkstatus_get_param(NULL,
+                            TRY_DIFF_FOR_CONSENSUS_NEWER_NAME,
+                            DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER,
+                            MIN_TRY_DIFF_FOR_CONSENSUS_NEWER,
+                            MAX_TRY_DIFF_FOR_CONSENSUS_NEWER);
+
+  if (flav != -1) {
+    /* IF we have a parsed consensus of this type, we can do an
+     * if-modified-time based on it. */
+    networkstatus_t *v;
+    v = networkstatus_get_latest_consensus_by_flavor(flav);
+    if (v) {
+      /* In networks with particularly short V3AuthVotingIntervals,
+       * ask for the consensus if it's been modified since half the
+       * V3AuthVotingInterval of the most recent consensus. */
+      time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
+      if (v->fresh_until > v->valid_after
+          && ims_delay > (v->fresh_until - v->valid_after)/2) {
+        ims_delay = (v->fresh_until - v->valid_after)/2;
+      }
+      if_modified_since = v->valid_after + ims_delay;
+      if (v->valid_after >= approx_time() - max_age_for_diff) {
+        memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN);
+        or_diff_from_is_set = 1;
+      }
+    }
+  } else {
+    /* Otherwise it might be a consensus we don't parse, but which we
+     * do cache.  Look at the cached copy, perhaps. */
+    cached_dir_t *cd = dirserv_get_consensus(resource);
+    /* We have no method of determining the voting interval from an
+     * unparsed consensus, so we use the default. */
+    if (cd) {
+      if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
+      if (cd->published >= approx_time() - max_age_for_diff) {
+        memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN);
+        or_diff_from_is_set = 1;
+      }
+    }
+  }
+
+  if (if_modified_since > 0)
+    directory_request_set_if_modified_since(req, if_modified_since);
+  if (or_diff_from_is_set) {
+    char hex[HEX_DIGEST256_LEN + 1];
+    base16_encode(hex, sizeof(hex),
+                  (const char*)or_diff_from, sizeof(or_diff_from));
+    directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex);
+  }
+}
+
+/** Start a connection to a random running directory server, using
+ * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
+ * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
+ * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
+ * or router_pick_trusteddirserver().
+ */
+MOCK_IMPL(void,
+directory_get_from_dirserver,(
+                            uint8_t dir_purpose,
+                            uint8_t router_purpose,
+                            const char *resource,
+                            int pds_flags,
+                            download_want_authority_t want_authority))
+{
+  const routerstatus_t *rs = NULL;
+  const or_options_t *options = get_options();
+  int prefer_authority = (directory_fetches_from_authorities(options)
+                          || want_authority == DL_WANT_AUTHORITY);
+  int require_authority = 0;
+  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose,
+                                            resource);
+  dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource);
+
+  if (type == NO_DIRINFO)
+    return;
+
+  if (!options->FetchServerDescriptors)
+    return;
+
+  circuit_guard_state_t *guard_state = NULL;
+  if (!get_via_tor) {
+    if (options->UseBridges && !(type & BRIDGE_DIRINFO)) {
+      /* We want to ask a running bridge for which we have a descriptor.
+       *
+       * When we ask choose_random_entry() for a bridge, we specify what
+       * sort of dir fetch we'll be doing, so it won't return a bridge
+       * that can't answer our question.
+       */
+      const node_t *node = guards_choose_dirguard(dir_purpose, &guard_state);
+      if (node && node->ri) {
+        /* every bridge has a routerinfo. */
+        routerinfo_t *ri = node->ri;
+        /* clients always make OR connections to bridges */
+        tor_addr_port_t or_ap;
+        directory_request_t *req = directory_request_new(dir_purpose);
+        /* we are willing to use a non-preferred address if we need to */
+        fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
+                                             &or_ap);
+        directory_request_set_or_addr_port(req, &or_ap);
+        directory_request_set_directory_id_digest(req,
+                                            ri->cache_info.identity_digest);
+        directory_request_set_router_purpose(req, router_purpose);
+        directory_request_set_resource(req, resource);
+        if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
+          dir_consensus_request_set_additional_headers(req, resource);
+        directory_request_set_guard_state(req, guard_state);
+        directory_initiate_request(req);
+        directory_request_free(req);
+      } else {
+        if (guard_state) {
+          entry_guard_cancel(&guard_state);
+        }
+        log_notice(LD_DIR, "Ignoring directory request, since no bridge "
+                           "nodes are available yet.");
+      }
+
+      return;
+    } else {
+      if (prefer_authority || (type & BRIDGE_DIRINFO)) {
+        /* only ask authdirservers, and don't ask myself */
+        rs = router_pick_trusteddirserver(type, pds_flags);
+        if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                                        PDS_NO_EXISTING_MICRODESC_FETCH))) {
+          /* We don't want to fetch from any authorities that we're currently
+           * fetching server descriptors from, and we got no match.  Did we
+           * get no match because all the authorities have connections
+           * fetching server descriptors (in which case we should just
+           * return,) or because all the authorities are down or on fire or
+           * unreachable or something (in which case we should go on with
+           * our fallback code)? */
+          pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH);
+          rs = router_pick_trusteddirserver(type, pds_flags);
+          if (rs) {
+            log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
+                      "are in use.");
+            return;
+          }
+        }
+        if (rs == NULL && require_authority) {
+          log_info(LD_DIR, "No authorities were available for %s: will try "
+                   "later.", dir_conn_purpose_to_string(dir_purpose));
+          return;
+        }
+      }
+      if (!rs && !(type & BRIDGE_DIRINFO)) {
+        rs = directory_pick_generic_dirserver(type, pds_flags,
+                                              dir_purpose,
+                                              &guard_state);
+        if (!rs)
+          get_via_tor = 1; /* last resort: try routing it via Tor */
+      }
+    }
+  }
+
+  if (get_via_tor) {
+    /* Never use fascistfirewall; we're going via Tor. */
+    pds_flags |= PDS_IGNORE_FASCISTFIREWALL;
+    rs = router_pick_directory_server(type, pds_flags);
+  }
+
+  /* If we have any hope of building an indirect conn, we know some router
+   * descriptors.  If (rs==NULL), we can't build circuits anyway, so
+   * there's no point in falling back to the authorities in this case. */
+  if (rs) {
+    const dir_indirection_t indirection =
+      get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP;
+    directory_request_t *req = directory_request_new(dir_purpose);
+    directory_request_set_routerstatus(req, rs);
+    directory_request_set_router_purpose(req, router_purpose);
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
+      dir_consensus_request_set_additional_headers(req, resource);
+    if (guard_state)
+      directory_request_set_guard_state(req, guard_state);
+    directory_initiate_request(req);
+    directory_request_free(req);
+  } else {
+    log_notice(LD_DIR,
+               "While fetching directory info, "
+               "no running dirservers known. Will try again later. "
+               "(purpose %d)", dir_purpose);
+    if (!purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
+      /* remember we tried them all and failed. */
+      directory_all_unreachable(time(NULL));
+    }
+  }
+}
+
+/** As directory_get_from_dirserver, but initiates a request to <i>every</i>
+ * directory authority other than ourself.  Only for use by authorities when
+ * searching for missing information while voting. */
+void
+directory_get_from_all_authorities(uint8_t dir_purpose,
+                                   uint8_t router_purpose,
+                                   const char *resource)
+{
+  tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
+             dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
+
+  SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
+                          dir_server_t *, ds) {
+      if (router_digest_is_me(ds->digest))
+        continue;
+      if (!(ds->type & V3_DIRINFO))
+        continue;
+      const routerstatus_t *rs = &ds->fake_status;
+      directory_request_t *req = directory_request_new(dir_purpose);
+      directory_request_set_routerstatus(req, rs);
+      directory_request_set_router_purpose(req, router_purpose);
+      directory_request_set_resource(req, resource);
+      directory_initiate_request(req);
+      directory_request_free(req);
+  } SMARTLIST_FOREACH_END(ds);
+}
+
+/** Return true iff <b>ind</b> requires a multihop circuit. */
+static int
+dirind_is_anon(dir_indirection_t ind)
+{
+  return ind == DIRIND_ANON_DIRPORT || ind == DIRIND_ANONYMOUS;
+}
+
+/* Choose reachable OR and Dir addresses and ports from status, copying them
+ * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're
+ * connecting via another relay, so choose the primary IPv4 address and ports.
+ *
+ * status should have at least one reachable address, if we can't choose a
+ * reachable address, warn and return -1. Otherwise, return 0.
+ */
+static int
+directory_choose_address_routerstatus(const routerstatus_t *status,
+                                      dir_indirection_t indirection,
+                                      tor_addr_port_t *use_or_ap,
+                                      tor_addr_port_t *use_dir_ap)
+{
+  tor_assert(status != NULL);
+  tor_assert(use_or_ap != NULL);
+  tor_assert(use_dir_ap != NULL);
+
+  const or_options_t *options = get_options();
+  int have_or = 0, have_dir = 0;
+
+  /* We expect status to have at least one reachable address if we're
+   * connecting to it directly.
+   *
+   * Therefore, we can simply use the other address if the one we want isn't
+   * allowed by the firewall.
+   *
+   * (When Tor uploads and downloads a hidden service descriptor, it uses
+   * DIRIND_ANONYMOUS, except for Tor2Web, which uses DIRIND_ONEHOP.
+   * So this code will only modify the address for Tor2Web's HS descriptor
+   * fetches. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS, to avoid
+   * HSDirs denying service by rejecting descriptors.)
+   */
+
+  /* Initialise the OR / Dir addresses */
+  tor_addr_make_null(&use_or_ap->addr, AF_UNSPEC);
+  use_or_ap->port = 0;
+  tor_addr_make_null(&use_dir_ap->addr, AF_UNSPEC);
+  use_dir_ap->port = 0;
+
+  /* ORPort connections */
+  if (indirection == DIRIND_ANONYMOUS) {
+    if (status->addr) {
+      /* Since we're going to build a 3-hop circuit and ask the 2nd relay
+       * to extend to this address, always use the primary (IPv4) OR address */
+      tor_addr_from_ipv4h(&use_or_ap->addr, status->addr);
+      use_or_ap->port = status->or_port;
+      have_or = 1;
+    }
+  } else if (indirection == DIRIND_ONEHOP) {
+    /* We use an IPv6 address if we have one and we prefer it.
+     * Use the preferred address and port if they are reachable, otherwise,
+     * use the alternate address and port (if any).
+     */
+    fascist_firewall_choose_address_rs(status, FIREWALL_OR_CONNECTION, 0,
+                                       use_or_ap);
+    have_or = tor_addr_port_is_valid_ap(use_or_ap, 0);
+  }
+
+  /* DirPort connections
+   * DIRIND_ONEHOP uses ORPort, but may fall back to the DirPort on relays */
+  if (indirection == DIRIND_DIRECT_CONN ||
+      indirection == DIRIND_ANON_DIRPORT ||
+      (indirection == DIRIND_ONEHOP
+       && !directory_must_use_begindir(options))) {
+    fascist_firewall_choose_address_rs(status, FIREWALL_DIR_CONNECTION, 0,
+                                       use_dir_ap);
+    have_dir = tor_addr_port_is_valid_ap(use_dir_ap, 0);
+  }
+
+  /* We rejected all addresses in the relay's status. This means we can't
+   * connect to it. */
+  if (!have_or && !have_dir) {
+    static int logged_backtrace = 0;
+    log_info(LD_BUG, "Rejected all OR and Dir addresses from %s when "
+             "launching an outgoing directory connection to: IPv4 %s OR %d "
+             "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status),
+             fmt_addr32(status->addr), status->or_port,
+             status->dir_port, fmt_addr(&status->ipv6_addr),
+             status->ipv6_orport, status->dir_port);
+    if (!logged_backtrace) {
+      log_backtrace(LOG_INFO, LD_BUG, "Addresses came from");
+      logged_backtrace = 1;
+    }
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Return true iff <b>conn</b> is the client side of a directory connection
+ * we launched to ourself in order to determine the reachability of our
+ * dir_port. */
+static int
+directory_conn_is_self_reachability_test(dir_connection_t *conn)
+{
+  if (conn->requested_resource &&
+      !strcmpstart(conn->requested_resource,"authority")) {
+    const routerinfo_t *me = router_get_my_routerinfo();
+    if (me &&
+        router_digest_is_me(conn->identity_digest) &&
+        tor_addr_eq_ipv4h(&conn->base_.addr, me->addr) && /*XXXX prop 118*/
+        me->dir_port == conn->base_.port)
+      return 1;
+  }
+  return 0;
+}
+
+/** Called when we are unable to complete the client's request to a directory
+ * server due to a network error: Mark the router as down and try again if
+ * possible.
+ */
+static void
+connection_dir_request_failed(dir_connection_t *conn)
+{
+  if (conn->guard_state) {
+    /* We haven't seen a success on this guard state, so consider it to have
+     * failed. */
+    entry_guard_failed(&conn->guard_state);
+  }
+  if (directory_conn_is_self_reachability_test(conn)) {
+    return; /* this was a test fetch. don't retry. */
+  }
+  if (!entry_list_is_constrained(get_options()))
+    router_set_status(conn->identity_digest, 0); /* don't try this one again */
+  if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
+    log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from "
+             "directory server at '%s'; retrying",
+             conn->base_.address);
+    if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
+      connection_dir_bridge_routerdesc_failed(conn);
+    connection_dir_download_routerdesc_failed(conn);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
+    if (conn->requested_resource)
+      networkstatus_consensus_download_failed(0, conn->requested_resource);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
+    log_info(LD_DIR, "Giving up on certificate fetch from directory server "
+             "at '%s'; retrying",
+             conn->base_.address);
+    connection_dir_download_cert_failed(conn, 0);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
+    log_info(LD_DIR, "Giving up downloading detached signatures from '%s'",
+             conn->base_.address);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
+    log_info(LD_DIR, "Giving up downloading votes from '%s'",
+             conn->base_.address);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    log_info(LD_DIR, "Giving up on downloading microdescriptors from "
+             "directory server at '%s'; will retry", conn->base_.address);
+    connection_dir_download_routerdesc_failed(conn);
+  }
+}
+
+/** Helper: Attempt to fetch directly the descriptors of each bridge
+ * listed in <b>failed</b>.
+ */
+static void
+connection_dir_retry_bridges(smartlist_t *descs)
+{
+  char digest[DIGEST_LEN];
+  SMARTLIST_FOREACH(descs, const char *, cp,
+  {
+    if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
+      log_warn(LD_BUG, "Malformed fingerprint in list: %s",
+              escaped(cp));
+      continue;
+    }
+    retry_bridge_descriptor_fetch_directly(digest);
+  });
+}
+
+/** Called when an attempt to download one or more router descriptors
+ * or extra-info documents on connection <b>conn</b> failed.
+ */
+static void
+connection_dir_download_routerdesc_failed(dir_connection_t *conn)
+{
+  /* No need to increment the failure count for routerdescs, since
+   * it's not their fault. */
+
+  /* No need to relaunch descriptor downloads here: we already do it
+   * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+
+  (void) conn;
+}
+
+/** Called when an attempt to download a bridge's routerdesc from
+ * one of the authorities failed due to a network error. If
+ * possible attempt to download descriptors from the bridge directly.
+ */
+static void
+connection_dir_bridge_routerdesc_failed(dir_connection_t *conn)
+{
+  smartlist_t *which = NULL;
+
+  /* Requests for bridge descriptors are in the form 'fp/', so ignore
+     anything else. */
+  if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/"))
+    return;
+
+  which = smartlist_new();
+  dir_split_resource_into_fingerprints(conn->requested_resource
+                                        + strlen("fp/"),
+                                       which, NULL, 0);
+
+  tor_assert(conn->base_.purpose != DIR_PURPOSE_FETCH_EXTRAINFO);
+  if (smartlist_len(which)) {
+    connection_dir_retry_bridges(which);
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+  }
+  smartlist_free(which);
+}
+
+/** Called when an attempt to fetch a certificate fails. */
+static void
+connection_dir_download_cert_failed(dir_connection_t *conn, int status)
+{
+  const char *fp_pfx = "fp/";
+  const char *fpsk_pfx = "fp-sk/";
+  smartlist_t *failed;
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
+
+  if (!conn->requested_resource)
+    return;
+  failed = smartlist_new();
+  /*
+   * We have two cases download by fingerprint (resource starts
+   * with "fp/") or download by fingerprint/signing key pair
+   * (resource starts with "fp-sk/").
+   */
+  if (!strcmpstart(conn->requested_resource, fp_pfx)) {
+    /* Download by fingerprint case */
+    dir_split_resource_into_fingerprints(conn->requested_resource +
+                                         strlen(fp_pfx),
+                                         failed, NULL, DSR_HEX);
+    SMARTLIST_FOREACH_BEGIN(failed, char *, cp) {
+      /* Null signing key digest indicates download by fp only */
+      authority_cert_dl_failed(cp, NULL, status);
+      tor_free(cp);
+    } SMARTLIST_FOREACH_END(cp);
+  } else if (!strcmpstart(conn->requested_resource, fpsk_pfx)) {
+    /* Download by (fp,sk) pairs */
+    dir_split_resource_into_fingerprint_pairs(conn->requested_resource +
+                                              strlen(fpsk_pfx), failed);
+    SMARTLIST_FOREACH_BEGIN(failed, fp_pair_t *, cp) {
+      authority_cert_dl_failed(cp->first, cp->second, status);
+      tor_free(cp);
+    } SMARTLIST_FOREACH_END(cp);
+  } else {
+    log_warn(LD_DIR,
+             "Don't know what to do with failure for cert fetch %s",
+             conn->requested_resource);
+  }
+
+  smartlist_free(failed);
+
+  update_certificate_downloads(time(NULL));
+}
+
+/* Should this tor instance only use begindir for all its directory requests?
+ */
+int
+directory_must_use_begindir(const or_options_t *options)
+{
+  /* Clients, onion services, and bridges must use begindir,
+   * relays and authorities do not have to */
+  return !public_server_mode(options);
+}
+
+/** Evaluate the situation and decide if we should use an encrypted
+ * "begindir-style" connection for this directory request.
+ * 0) If there is no DirPort, yes.
+ * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
+ *    or we're a dir mirror, no.
+ * 2) If we prefer to avoid begindir conns, and we're not fetching or
+ *    publishing a bridge relay descriptor, no.
+ * 3) Else yes.
+ * If returning 0, return in *reason why we can't use begindir.
+ * reason must not be NULL.
+ */
+static int
+directory_command_should_use_begindir(const or_options_t *options,
+                                      const directory_request_t *req,
+                                      const char **reason)
+{
+  const tor_addr_t *or_addr = &req->or_addr_port.addr;
+  //const tor_addr_t *dir_addr = &req->dir_addr_port.addr;
+  const int or_port = req->or_addr_port.port;
+  const int dir_port = req->dir_addr_port.port;
+
+  const dir_indirection_t indirection = req->indirection;
+
+  tor_assert(reason);
+  *reason = NULL;
+
+  /* Reasons why we must use begindir */
+  if (!dir_port) {
+    *reason = "(using begindir - directory with no DirPort)";
+    return 1; /* We don't know a DirPort -- must begindir. */
+  }
+  /* Reasons why we can't possibly use begindir */
+  if (!or_port) {
+    *reason = "directory with unknown ORPort";
+    return 0; /* We don't know an ORPort -- no chance. */
+  }
+  if (indirection == DIRIND_DIRECT_CONN ||
+      indirection == DIRIND_ANON_DIRPORT) {
+    *reason = "DirPort connection";
+    return 0;
+  }
+  if (indirection == DIRIND_ONEHOP) {
+    /* We're firewalled and want a direct OR connection */
+    if (!fascist_firewall_allows_address_addr(or_addr, or_port,
+                                              FIREWALL_OR_CONNECTION, 0, 0)) {
+      *reason = "ORPort not reachable";
+      return 0;
+    }
+  }
+  /* Reasons why we want to avoid using begindir */
+  if (indirection == DIRIND_ONEHOP) {
+    if (!directory_must_use_begindir(options)) {
+      *reason = "in relay mode";
+      return 0;
+    }
+  }
+  /* DIRIND_ONEHOP on a client, or DIRIND_ANONYMOUS
+   */
+  *reason = "(using begindir)";
+  return 1;
+}
+
+/**
+ * Create and return a new directory_request_t with purpose
+ * <b>dir_purpose</b>.
+ */
+directory_request_t *
+directory_request_new(uint8_t dir_purpose)
+{
+  tor_assert(dir_purpose >= DIR_PURPOSE_MIN_);
+  tor_assert(dir_purpose <= DIR_PURPOSE_MAX_);
+  tor_assert(dir_purpose != DIR_PURPOSE_SERVER);
+  tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2);
+  tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_HSDESC);
+
+  directory_request_t *result = tor_malloc_zero(sizeof(*result));
+  tor_addr_make_null(&result->or_addr_port.addr, AF_INET);
+  result->or_addr_port.port = 0;
+  tor_addr_make_null(&result->dir_addr_port.addr, AF_INET);
+  result->dir_addr_port.port = 0;
+  result->dir_purpose = dir_purpose;
+  result->router_purpose = ROUTER_PURPOSE_GENERAL;
+  result->indirection = DIRIND_ONEHOP;
+  return result;
+}
+/**
+ * Release all resources held by <b>req</b>.
+ */
+void
+directory_request_free_(directory_request_t *req)
+{
+  if (req == NULL)
+    return;
+  config_free_lines(req->additional_headers);
+  tor_free(req);
+}
+/**
+ * Set the address and OR port to use for this directory request.  If there is
+ * no OR port, we'll have to connect over the dirport.  (If there are both,
+ * the indirection setting determines which to use.)
+ */
+void
+directory_request_set_or_addr_port(directory_request_t *req,
+                                   const tor_addr_port_t *p)
+{
+  memcpy(&req->or_addr_port, p, sizeof(*p));
+}
+/**
+ * Set the address and dirport to use for this directory request.  If there
+ * is no dirport, we'll have to connect over the OR port.  (If there are both,
+ * the indirection setting determines which to use.)
+ */
+void
+directory_request_set_dir_addr_port(directory_request_t *req,
+                                    const tor_addr_port_t *p)
+{
+  memcpy(&req->dir_addr_port, p, sizeof(*p));
+}
+/**
+ * Set the RSA identity digest of the directory to use for this directory
+ * request.
+ */
+void
+directory_request_set_directory_id_digest(directory_request_t *req,
+                                          const char *digest)
+{
+  memcpy(req->digest, digest, DIGEST_LEN);
+}
+/**
+ * Set the router purpose associated with uploaded and downloaded router
+ * descriptors and extrainfo documents in this directory request.  The purpose
+ * must be one of ROUTER_PURPOSE_GENERAL (the default) or
+ * ROUTER_PURPOSE_BRIDGE.
+ */
+void
+directory_request_set_router_purpose(directory_request_t *req,
+                                     uint8_t router_purpose)
+{
+  tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL ||
+             router_purpose == ROUTER_PURPOSE_BRIDGE);
+  // assert that it actually makes sense to set this purpose, given
+  // the dir_purpose.
+  req->router_purpose = router_purpose;
+}
+/**
+ * Set the indirection to be used for the directory request.  The indirection
+ * parameter configures whether to connect to a DirPort or ORPort, and whether
+ * to anonymize the connection.  DIRIND_ONEHOP (use ORPort, don't anonymize)
+ * is the default.  See dir_indirection_t for more information.
+ */
+void
+directory_request_set_indirection(directory_request_t *req,
+                                  dir_indirection_t indirection)
+{
+  req->indirection = indirection;
+}
+
+/**
+ * Set a pointer to the resource to request from a directory.  Different
+ * request types use resources to indicate different components of their URL.
+ * Note that only an alias to <b>resource</b> is stored, so the
+ * <b>resource</b> must outlive the request.
+ */
+void
+directory_request_set_resource(directory_request_t *req,
+                               const char *resource)
+{
+  req->resource = resource;
+}
+/**
+ * Set a pointer to the payload to include with this directory request, along
+ * with its length.  Note that only an alias to <b>payload</b> is stored, so
+ * the <b>payload</b> must outlive the request.
+ */
+void
+directory_request_set_payload(directory_request_t *req,
+                              const char *payload,
+                              size_t payload_len)
+{
+  tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose));
+
+  req->payload = payload;
+  req->payload_len = payload_len;
+}
+/**
+ * Set an if-modified-since date to send along with the request.  The
+ * default is 0 (meaning, send no if-modified-since header).
+ */
+void
+directory_request_set_if_modified_since(directory_request_t *req,
+                                        time_t if_modified_since)
+{
+  req->if_modified_since = if_modified_since;
+}
+
+/** Include a header of name <b>key</b> with content <b>val</b> in the
+ * request. Neither may include newlines or other odd characters. Their
+ * ordering is not currently guaranteed.
+ *
+ * Note that, as elsewhere in this module, header keys include a trailing
+ * colon and space.
+ */
+void
+directory_request_add_header(directory_request_t *req,
+                             const char *key,
+                             const char *val)
+{
+  config_line_prepend(&req->additional_headers, key, val);
+}
+/**
+ * Set an object containing HS data to be associated with this request.  Note
+ * that only an alias to <b>query</b> is stored, so the <b>query</b> object
+ * must outlive the request.
+ */
+void
+directory_request_set_rend_query(directory_request_t *req,
+                                 const rend_data_t *query)
+{
+  if (query) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 ||
+               req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+  }
+  req->rend_query = query;
+}
+/**
+ * Set an object containing HS connection identifier to be associated with
+ * this request. Note that only an alias to <b>ident</b> is stored, so the
+ * <b>ident</b> object must outlive the request.
+ */
+void
+directory_request_upload_set_hs_ident(directory_request_t *req,
+                                      const hs_ident_dir_conn_t *ident)
+{
+  if (ident) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_UPLOAD_HSDESC);
+  }
+  req->hs_ident = ident;
+}
+/**
+ * Set an object containing HS connection identifier to be associated with
+ * this fetch request. Note that only an alias to <b>ident</b> is stored, so
+ * the <b>ident</b> object must outlive the request.
+ */
+void
+directory_request_fetch_set_hs_ident(directory_request_t *req,
+                                     const hs_ident_dir_conn_t *ident)
+{
+  if (ident) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_HSDESC);
+  }
+  req->hs_ident = ident;
+}
+/** Set a static circuit_guard_state_t object to affliate with the request in
+ * <b>req</b>.  This object will receive notification when the attempt to
+ * connect to the guard either succeeds or fails. */
+void
+directory_request_set_guard_state(directory_request_t *req,
+                                  circuit_guard_state_t *state)
+{
+  req->guard_state = state;
+}
+
+/**
+ * Internal: Return true if any information for contacting the directory in
+ * <b>req</b> has been set, other than by the routerstatus. */
+static int
+directory_request_dir_contact_info_specified(const directory_request_t *req)
+{
+  /* We only check for ports here, since we don't use an addr unless the port
+   * is set */
+  return (req->or_addr_port.port ||
+          req->dir_addr_port.port ||
+          ! tor_digest_is_zero(req->digest));
+}
+
+/**
+ * Set the routerstatus to use for the directory associated with this
+ * request.  If this option is set, then no other function to set the
+ * directory's address or identity should be called.
+ */
+void
+directory_request_set_routerstatus(directory_request_t *req,
+                                   const routerstatus_t *status)
+{
+  req->routerstatus = status;
+}
+/**
+ * Helper: update the addresses, ports, and identities in <b>req</b>
+ * from the routerstatus object in <b>req</b>.  Return 0 on success.
+ * On failure, warn and return -1.
+ */
+static int
+directory_request_set_dir_from_routerstatus(directory_request_t *req)
+
+{
+  const routerstatus_t *status = req->routerstatus;
+  if (BUG(status == NULL))
+    return -1;
+  const or_options_t *options = get_options();
+  const node_t *node;
+  tor_addr_port_t use_or_ap, use_dir_ap;
+  const int anonymized_connection = dirind_is_anon(req->indirection);
+
+  tor_assert(status != NULL);
+
+  node = node_get_by_id(status->identity_digest);
+
+  /* XXX The below check is wrong: !node means it's not in the consensus,
+   * but we haven't checked if we have a descriptor for it -- and also,
+   * we only care about the descriptor if it's a begindir-style anonymized
+   * connection. */
+  if (!node && anonymized_connection) {
+    log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
+             "don't have its router descriptor.",
+             routerstatus_describe(status));
+    return -1;
+  }
+
+  if (options->ExcludeNodes && options->StrictNodes &&
+      routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
+    log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
+             "it's in our ExcludedNodes list and StrictNodes is set. "
+             "Skipping. This choice might make your Tor not work.",
+             routerstatus_describe(status),
+             dir_conn_purpose_to_string(req->dir_purpose));
+    return -1;
+  }
+
+    /* At this point, if we are a client making a direct connection to a
+   * directory server, we have selected a server that has at least one address
+   * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
+   * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
+   * possible. (If UseBridges is set, clients always use IPv6, and prefer it
+   * by default.)
+   *
+   * Now choose an address that we can use to connect to the directory server.
+   */
+  if (directory_choose_address_routerstatus(status,
+                                            req->indirection, &use_or_ap,
+                                            &use_dir_ap) < 0) {
+    return -1;
+  }
+
+  directory_request_set_or_addr_port(req, &use_or_ap);
+  directory_request_set_dir_addr_port(req, &use_dir_ap);
+  directory_request_set_directory_id_digest(req, status->identity_digest);
+  return 0;
+}
+
+/**
+ * Launch the provided directory request, configured in <b>request</b>.
+ * After this function is called, you can free <b>request</b>.
+ */
+MOCK_IMPL(void,
+directory_initiate_request,(directory_request_t *request))
+{
+  tor_assert(request);
+  if (request->routerstatus) {
+    tor_assert_nonfatal(
+               ! directory_request_dir_contact_info_specified(request));
+    if (directory_request_set_dir_from_routerstatus(request) < 0) {
+      return;
+    }
+  }
+
+  const tor_addr_port_t *or_addr_port = &request->or_addr_port;
+  const tor_addr_port_t *dir_addr_port = &request->dir_addr_port;
+  const char *digest = request->digest;
+  const uint8_t dir_purpose = request->dir_purpose;
+  const uint8_t router_purpose = request->router_purpose;
+  const dir_indirection_t indirection = request->indirection;
+  const char *resource = request->resource;
+  const rend_data_t *rend_query = request->rend_query;
+  const hs_ident_dir_conn_t *hs_ident = request->hs_ident;
+  circuit_guard_state_t *guard_state = request->guard_state;
+
+  tor_assert(or_addr_port->port || dir_addr_port->port);
+  tor_assert(digest);
+
+  dir_connection_t *conn;
+  const or_options_t *options = get_options();
+  int socket_error = 0;
+  const char *begindir_reason = NULL;
+  /* Should the connection be to a relay's OR port (and inside that we will
+   * send our directory request)? */
+  const int use_begindir =
+    directory_command_should_use_begindir(options, request, &begindir_reason);
+
+  /* Will the connection go via a three-hop Tor circuit? Note that this
+   * is separate from whether it will use_begindir. */
+  const int anonymized_connection = dirind_is_anon(indirection);
+
+  /* What is the address we want to make the directory request to? If
+   * we're making a begindir request this is the ORPort of the relay
+   * we're contacting; if not a begindir request, this is its DirPort.
+   * Note that if anonymized_connection is true, we won't be initiating
+   * a connection directly to this address. */
+  tor_addr_t addr;
+  tor_addr_copy(&addr, &(use_begindir ? or_addr_port : dir_addr_port)->addr);
+  uint16_t port = (use_begindir ? or_addr_port : dir_addr_port)->port;
+
+  log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
+            anonymized_connection, use_begindir);
+
+  log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
+
+  if (purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
+    tor_assert(anonymized_connection ||
+               rend_non_anonymous_mode_enabled(options));
+  }
+
+  /* use encrypted begindir connections for everything except relays
+   * this provides better protection for directory fetches */
+  if (!use_begindir && directory_must_use_begindir(options)) {
+    log_warn(LD_BUG, "Client could not use begindir connection: %s",
+             begindir_reason ? begindir_reason : "(NULL)");
+    return;
+  }
+
+  /* ensure that we don't make direct connections when a SOCKS server is
+   * configured. */
+  if (!anonymized_connection && !use_begindir && !options->HTTPProxy &&
+      (options->Socks4Proxy || options->Socks5Proxy)) {
+    log_warn(LD_DIR, "Cannot connect to a directory server through a "
+                     "SOCKS proxy!");
+    return;
+  }
+
+  /* Make sure that the destination addr and port we picked is viable. */
+  if (!port || tor_addr_is_null(&addr)) {
+    static int logged_backtrace = 0;
+    log_warn(LD_DIR,
+             "Cannot make an outgoing %sconnection without a remote %sPort.",
+             use_begindir ? "begindir " : "",
+             use_begindir ? "OR" : "Dir");
+    if (!logged_backtrace) {
+      log_backtrace(LOG_INFO, LD_BUG, "Address came from");
+      logged_backtrace = 1;
+    }
+    return;
+  }
+
+  conn = dir_connection_new(tor_addr_family(&addr));
+
+  /* set up conn so it's got all the data we need to remember */
+  tor_addr_copy(&conn->base_.addr, &addr);
+  conn->base_.port = port;
+  conn->base_.address = tor_addr_to_str_dup(&addr);
+  memcpy(conn->identity_digest, digest, DIGEST_LEN);
+
+  conn->base_.purpose = dir_purpose;
+  conn->router_purpose = router_purpose;
+
+  /* give it an initial state */
+  conn->base_.state = DIR_CONN_STATE_CONNECTING;
+
+  /* decide whether we can learn our IP address from this conn */
+  /* XXXX This is a bad name for this field now. */
+  conn->dirconn_direct = !anonymized_connection;
+
+  /* copy rendezvous data, if any */
+  if (rend_query) {
+    /* We can't have both v2 and v3+ identifier. */
+    tor_assert_nonfatal(!hs_ident);
+    conn->rend_data = rend_data_dup(rend_query);
+  }
+  if (hs_ident) {
+    /* We can't have both v2 and v3+ identifier. */
+    tor_assert_nonfatal(!rend_query);
+    conn->hs_ident = hs_ident_dir_conn_dup(hs_ident);
+  }
+
+  if (!anonymized_connection && !use_begindir) {
+    /* then we want to connect to dirport directly */
+
+    if (options->HTTPProxy) {
+      tor_addr_copy(&addr, &options->HTTPProxyAddr);
+      port = options->HTTPProxyPort;
+    }
+
+    // In this case we should not have picked a directory guard.
+    if (BUG(guard_state)) {
+      entry_guard_cancel(&guard_state);
+    }
+
+    switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr,
+                               port, &socket_error)) {
+      case -1:
+        connection_mark_for_close(TO_CONN(conn));
+        return;
+      case 1:
+        /* start flushing conn */
+        conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
+        /* fall through */
+      case 0:
+        /* queue the command on the outbuf */
+        directory_send_command(conn, 1, request);
+        connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
+        /* writable indicates finish, readable indicates broken link,
+           error indicates broken link in windowsland. */
+    }
+  } else {
+    /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with
+     * begindir, maybe not with begindir) */
+
+    entry_connection_t *linked_conn;
+
+    /* Anonymized tunneled connections can never share a circuit.
+     * One-hop directory connections can share circuits with each other
+     * but nothing else. */
+    int iso_flags = anonymized_connection ? ISO_STREAM : ISO_SESSIONGRP;
+
+    /* If it's an anonymized connection, remember the fact that we
+     * wanted it for later: maybe we'll want it again soon. */
+    if (anonymized_connection && use_begindir)
+      rep_hist_note_used_internal(time(NULL), 0, 1);
+    else if (anonymized_connection && !use_begindir)
+      rep_hist_note_used_port(time(NULL), conn->base_.port);
+
+    // In this case we should not have a directory guard; we'll
+    // get a regular guard later when we build the circuit.
+    if (BUG(anonymized_connection && guard_state)) {
+      entry_guard_cancel(&guard_state);
+    }
+
+    conn->guard_state = guard_state;
+
+    /* make an AP connection
+     * populate it and add it at the right state
+     * hook up both sides
+     */
+    linked_conn =
+      connection_ap_make_link(TO_CONN(conn),
+                              conn->base_.address, conn->base_.port,
+                              digest,
+                              SESSION_GROUP_DIRCONN, iso_flags,
+                              use_begindir, !anonymized_connection);
+    if (!linked_conn) {
+      log_warn(LD_NET,"Making tunnel to dirserver failed.");
+      connection_mark_for_close(TO_CONN(conn));
+      return;
+    }
+
+    if (connection_add(TO_CONN(conn)) < 0) {
+      log_warn(LD_NET,"Unable to add connection for link to dirserver.");
+      connection_mark_for_close(TO_CONN(conn));
+      return;
+    }
+    conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
+    /* queue the command on the outbuf */
+    directory_send_command(conn, 0, request);
+
+    connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
+    connection_start_reading(ENTRY_TO_CONN(linked_conn));
+  }
+}
+
+/** Return true iff anything we say on <b>conn</b> is being encrypted before
+ * we send it to the client/server. */
+int
+connection_dir_is_encrypted(const dir_connection_t *conn)
+{
+  /* Right now it's sufficient to see if conn is or has been linked, since
+   * the only thing it could be linked to is an edge connection on a
+   * circuit, and the only way it could have been unlinked is at the edge
+   * connection getting closed.
+   */
+  return TO_CONN(conn)->linked;
+}
+
+/** Helper for sorting
+ *
+ * sort strings alphabetically
+ */
+static int
+compare_strs_(const void **a, const void **b)
+{
+  const char *s1 = *a, *s2 = *b;
+  return strcmp(s1, s2);
+}
+
+#define CONDITIONAL_CONSENSUS_FPR_LEN 3
+#if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
+#error "conditional consensus fingerprint length is larger than digest length"
+#endif
+
+/** Return the URL we should use for a consensus download.
+ *
+ * Use the "conditional consensus downloading" feature described in
+ * dir-spec.txt, i.e.
+ * GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
+ *
+ * If 'resource' is provided, it is the name of a consensus flavor to request.
+ */
+static char *
+directory_get_consensus_url(const char *resource)
+{
+  char *url = NULL;
+  const char *hyphen, *flavor;
+  if (resource==NULL || strcmp(resource, "ns")==0) {
+    flavor = ""; /* Request ns consensuses as "", so older servers will work*/
+    hyphen = "";
+  } else {
+    flavor = resource;
+    hyphen = "-";
+  }
+
+  {
+    char *authority_id_list;
+    smartlist_t *authority_digests = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
+                            dir_server_t *, ds) {
+        char *hex;
+        if (!(ds->type & V3_DIRINFO))
+          continue;
+
+        hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1);
+        base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1,
+                      ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN);
+        smartlist_add(authority_digests, hex);
+    } SMARTLIST_FOREACH_END(ds);
+    smartlist_sort(authority_digests, compare_strs_);
+    authority_id_list = smartlist_join_strings(authority_digests,
+                                               "+", 0, NULL);
+
+    tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
+                 hyphen, flavor, authority_id_list);
+
+    SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
+    smartlist_free(authority_digests);
+    tor_free(authority_id_list);
+  }
+  return url;
+}
+
+/**
+ * Copies the ipv6 from source to destination, subject to buffer size limit
+ * size. If decorate is true, makes sure the copied address is decorated.
+ */
+static void
+copy_ipv6_address(char* destination, const char* source, size_t len,
+                  int decorate) {
+  tor_assert(destination);
+  tor_assert(source);
+
+  if (decorate && source[0] != '[') {
+    tor_snprintf(destination, len, "[%s]", source);
+  } else {
+    strlcpy(destination, source, len);
+  }
+}
+
+/** Queue an appropriate HTTP command for <b>request</b> on
+ * <b>conn</b>-\>outbuf.  If <b>direct</b> is true, we're making a
+ * non-anonymized connection to the dirport.
+ */
+static void
+directory_send_command(dir_connection_t *conn,
+                       const int direct,
+                       const directory_request_t *req)
+{
+  tor_assert(req);
+  const int purpose = req->dir_purpose;
+  const char *resource = req->resource;
+  const char *payload = req->payload;
+  const size_t payload_len = req->payload_len;
+  const time_t if_modified_since = req->if_modified_since;
+  const int anonymized_connection = dirind_is_anon(req->indirection);
+
+  char proxystring[256];
+  char hoststring[128];
+  /* NEEDS to be the same size hoststring.
+   Will be decorated with brackets around it if it is ipv6. */
+  char decorated_address[128];
+  smartlist_t *headers = smartlist_new();
+  char *url;
+  char *accept_encoding;
+  size_t url_len;
+  char request[8192];
+  size_t request_len, total_request_len = 0;
+  const char *httpcommand = NULL;
+
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  tor_free(conn->requested_resource);
+  if (resource)
+    conn->requested_resource = tor_strdup(resource);
+
+  /* decorate the ip address if it is ipv6 */
+  if (strchr(conn->base_.address, ':')) {
+    copy_ipv6_address(decorated_address, conn->base_.address,
+                      sizeof(decorated_address), 1);
+  } else {
+    strlcpy(decorated_address, conn->base_.address, sizeof(decorated_address));
+  }
+
+  /* come up with a string for which Host: we want */
+  if (conn->base_.port == 80) {
+    strlcpy(hoststring, decorated_address, sizeof(hoststring));
+  } else {
+    tor_snprintf(hoststring, sizeof(hoststring), "%s:%d",
+                 decorated_address, conn->base_.port);
+  }
+
+  /* Format if-modified-since */
+  if (if_modified_since) {
+    char b[RFC1123_TIME_LEN+1];
+    format_rfc1123_time(b, if_modified_since);
+    smartlist_add_asprintf(headers, "If-Modified-Since: %s\r\n", b);
+  }
+
+  /* come up with some proxy lines, if we're using one. */
+  if (direct && get_options()->HTTPProxy) {
+    char *base64_authenticator=NULL;
+    const char *authenticator = get_options()->HTTPProxyAuthenticator;
+
+    tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
+    if (authenticator) {
+      base64_authenticator = alloc_http_authenticator(authenticator);
+      if (!base64_authenticator)
+        log_warn(LD_BUG, "Encoding http authenticator failed");
+    }
+    if (base64_authenticator) {
+      smartlist_add_asprintf(headers,
+                   "Proxy-Authorization: Basic %s\r\n",
+                   base64_authenticator);
+      tor_free(base64_authenticator);
+    }
+  } else {
+    proxystring[0] = 0;
+  }
+
+  if (! anonymized_connection) {
+    /* Add Accept-Encoding. */
+    accept_encoding = accept_encoding_header();
+    smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n",
+                           accept_encoding);
+    tor_free(accept_encoding);
+  }
+
+  /* Add additional headers, if any */
+  {
+    config_line_t *h;
+    for (h = req->additional_headers; h; h = h->next) {
+      smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value);
+    }
+  }
+
+  switch (purpose) {
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      /* resource is optional.  If present, it's a flavor name */
+      tor_assert(!payload);
+      httpcommand = "GET";
+      url = directory_get_consensus_url(resource);
+      log_info(LD_DIR, "Downloading consensus from %s using %s",
+               hoststring, url);
+      break;
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      tor_assert(resource);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/keys/%s", resource);
+      break;
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+      tor_assert(resource);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/status-vote/next/%s.z", resource);
+      break;
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+      tor_assert(!resource);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      url = tor_strdup("/tor/status-vote/next/consensus-signatures.z");
+      break;
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+      tor_assert(resource);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/server/%s", resource);
+      break;
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      tor_assert(resource);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/extra/%s", resource);
+      break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      tor_assert(resource);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/micro/%s", resource);
+      break;
+    case DIR_PURPOSE_UPLOAD_DIR: {
+      const char *why = router_get_descriptor_gen_reason();
+      tor_assert(!resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      url = tor_strdup("/tor/");
+      if (!why) {
+        why = "for no reason at all";
+      }
+      smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why);
+      break;
+    }
+    case DIR_PURPOSE_UPLOAD_VOTE:
+      tor_assert(!resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      url = tor_strdup("/tor/post/vote");
+      break;
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+      tor_assert(!resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      url = tor_strdup("/tor/post/consensus-signature");
+      break;
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+      tor_assert(resource);
+      tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/rendezvous2/%s", resource);
+      break;
+    case DIR_PURPOSE_FETCH_HSDESC:
+      tor_assert(resource);
+      tor_assert(strlen(resource) <= ED25519_BASE64_LEN);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/hs/3/%s", resource);
+      break;
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+      tor_assert(!resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      url = tor_strdup("/tor/rendezvous2/publish");
+      break;
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      tor_assert(resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      tor_asprintf(&url, "/tor/hs/%s/publish", resource);
+      break;
+    default:
+      tor_assert(0);
+      return;
+  }
+
+  /* warn in the non-tunneled case */
+  if (direct && (strlen(proxystring) + strlen(url) >= 4096)) {
+    log_warn(LD_BUG,
+             "Squid does not like URLs longer than 4095 bytes, and this "
+             "one is %d bytes long: %s%s",
+             (int)(strlen(proxystring) + strlen(url)), proxystring, url);
+  }
+
+  tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
+
+  request_len = strlen(request);
+  total_request_len += request_len;
+  connection_buf_add(request, request_len, TO_CONN(conn));
+
+  url_len = strlen(url);
+  total_request_len += url_len;
+  connection_buf_add(url, url_len, TO_CONN(conn));
+  tor_free(url);
+
+  if (!strcmp(httpcommand, "POST") || payload) {
+    smartlist_add_asprintf(headers, "Content-Length: %lu\r\n",
+                 payload ? (unsigned long)payload_len : 0);
+  }
+
+  {
+    char *header = smartlist_join_strings(headers, "", 0, NULL);
+    tor_snprintf(request, sizeof(request), " HTTP/1.0\r\nHost: %s\r\n%s\r\n",
+                 hoststring, header);
+    tor_free(header);
+  }
+
+  request_len = strlen(request);
+  total_request_len += request_len;
+  connection_buf_add(request, request_len, TO_CONN(conn));
+
+  if (payload) {
+    /* then send the payload afterwards too */
+    connection_buf_add(payload, payload_len, TO_CONN(conn));
+    total_request_len += payload_len;
+  }
+
+  SMARTLIST_FOREACH(headers, char *, h, tor_free(h));
+  smartlist_free(headers);
+
+  log_debug(LD_DIR,
+            "Sent request to directory server '%s:%d': "
+            "(purpose: %d, request size: %"TOR_PRIuSZ", "
+            "payload size: %"TOR_PRIuSZ")",
+            conn->base_.address, conn->base_.port,
+            conn->base_.purpose,
+            (total_request_len),
+            (payload ? payload_len : 0));
+}
+
+/** Parse an HTTP request string <b>headers</b> of the form
+ * \verbatim
+ * "\%s [http[s]://]\%s HTTP/1..."
+ * \endverbatim
+ * If it's well-formed, strdup the second \%s into *<b>url</b>, and
+ * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
+ * so it does. Return 0.
+ * Otherwise, return -1.
+ */
+STATIC int
+parse_http_url(const char *headers, char **url)
+{
+  char *command = NULL;
+  if (parse_http_command(headers, &command, url) < 0) {
+    return -1;
+  }
+  if (strcmpstart(*url, "/tor/")) {
+    char *new_url = NULL;
+    tor_asprintf(&new_url, "/tor%s%s",
+                 *url[0] == '/' ? "" : "/",
+                 *url);
+    tor_free(*url);
+    *url = new_url;
+  }
+  tor_free(command);
+  return 0;
+}
+
+/** Parse an HTTP request line at the start of a headers string.  On failure,
+ * return -1.  On success, set *<b>command_out</b> to a copy of the HTTP
+ * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
+ * return 0. */
+int
+parse_http_command(const char *headers, char **command_out, char **url_out)
+{
+  const char *command, *end_of_command;
+  char *s, *start, *tmp;
+
+  s = (char *)eat_whitespace_no_nl(headers);
+  if (!*s) return -1;
+  command = s;
+  s = (char *)find_whitespace(s); /* get past GET/POST */
+  if (!*s) return -1;
+  end_of_command = s;
+  s = (char *)eat_whitespace_no_nl(s);
+  if (!*s) return -1;
+  start = s; /* this is the URL, assuming it's valid */
+  s = (char *)find_whitespace(start);
+  if (!*s) return -1;
+
+  /* tolerate the http[s] proxy style of putting the hostname in the url */
+  if (s-start >= 4 && !strcmpstart(start,"http")) {
+    tmp = start + 4;
+    if (*tmp == 's')
+      tmp++;
+    if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
+      tmp = strchr(tmp+3, '/');
+      if (tmp && tmp < s) {
+        log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string");
+        start = tmp;
+      }
+    }
+  }
+
+  /* Check if the header is well formed (next sequence
+   * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */
+  {
+    unsigned minor_ver;
+    char ch;
+    char *e = (char *)eat_whitespace_no_nl(s);
+    if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) {
+      return -1;
+    }
+    if (ch != '\r')
+      return -1;
+  }
+
+  *url_out = tor_memdup_nulterm(start, s-start);
+  *command_out = tor_memdup_nulterm(command, end_of_command - command);
+  return 0;
+}
+
+/** Return a copy of the first HTTP header in <b>headers</b> whose key is
+ * <b>which</b>.  The key should be given with a terminating colon and space;
+ * this function copies everything after, up to but not including the
+ * following \\r\\n. */
+char *
+http_get_header(const char *headers, const char *which)
+{
+  const char *cp = headers;
+  while (cp) {
+    if (!strcasecmpstart(cp, which)) {
+      char *eos;
+      cp += strlen(which);
+      if ((eos = strchr(cp,'\r')))
+        return tor_strndup(cp, eos-cp);
+      else
+        return tor_strdup(cp);
+    }
+    cp = strchr(cp, '\n');
+    if (cp)
+      ++cp;
+  }
+  return NULL;
+}
+
+/** If <b>headers</b> indicates that a proxy was involved, then rewrite
+ * <b>conn</b>-\>address to describe our best guess of the address that
+ * originated this HTTP request. */
+static void
+http_set_address_origin(const char *headers, connection_t *conn)
+{
+  char *fwd;
+
+  fwd = http_get_header(headers, "Forwarded-For: ");
+  if (!fwd)
+    fwd = http_get_header(headers, "X-Forwarded-For: ");
+  if (fwd) {
+    tor_addr_t toraddr;
+    if (tor_addr_parse(&toraddr,fwd) == -1 ||
+        tor_addr_is_internal(&toraddr,0)) {
+      log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd));
+      tor_free(fwd);
+      return;
+    }
+
+    tor_free(conn->address);
+    conn->address = tor_strdup(fwd);
+    tor_free(fwd);
+  }
+}
+
+/** Parse an HTTP response string <b>headers</b> of the form
+ * \verbatim
+ * "HTTP/1.\%d \%d\%s\r\n...".
+ * \endverbatim
+ *
+ * If it's well-formed, assign the status code to *<b>code</b> and
+ * return 0.  Otherwise, return -1.
+ *
+ * On success: If <b>date</b> is provided, set *date to the Date
+ * header in the http headers, or 0 if no such header is found.  If
+ * <b>compression</b> is provided, set *<b>compression</b> to the
+ * compression method given in the Content-Encoding header, or 0 if no
+ * such header is found, or -1 if the value of the header is not
+ * recognized.  If <b>reason</b> is provided, strdup the reason string
+ * into it.
+ */
+int
+parse_http_response(const char *headers, int *code, time_t *date,
+                    compress_method_t *compression, char **reason)
+{
+  unsigned n1, n2;
+  char datestr[RFC1123_TIME_LEN+1];
+  smartlist_t *parsed_headers;
+  tor_assert(headers);
+  tor_assert(code);
+
+  while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */
+
+  if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 ||
+      (n1 != 0 && n1 != 1) ||
+      (n2 < 100 || n2 >= 600)) {
+    log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers));
+    return -1;
+  }
+  *code = n2;
+
+  parsed_headers = smartlist_new();
+  smartlist_split_string(parsed_headers, headers, "\n",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  if (reason) {
+    smartlist_t *status_line_elements = smartlist_new();
+    tor_assert(smartlist_len(parsed_headers));
+    smartlist_split_string(status_line_elements,
+                           smartlist_get(parsed_headers, 0),
+                           " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+    tor_assert(smartlist_len(status_line_elements) <= 3);
+    if (smartlist_len(status_line_elements) == 3) {
+      *reason = smartlist_get(status_line_elements, 2);
+      smartlist_set(status_line_elements, 2, NULL); /* Prevent free */
+    }
+    SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp));
+    smartlist_free(status_line_elements);
+  }
+  if (date) {
+    *date = 0;
+    SMARTLIST_FOREACH(parsed_headers, const char *, s,
+      if (!strcmpstart(s, "Date: ")) {
+        strlcpy(datestr, s+6, sizeof(datestr));
+        /* This will do nothing on failure, so we don't need to check
+           the result.   We shouldn't warn, since there are many other valid
+           date formats besides the one we use. */
+        parse_rfc1123_time(datestr, date);
+        break;
+      });
+  }
+  if (compression) {
+    const char *enc = NULL;
+    SMARTLIST_FOREACH(parsed_headers, const char *, s,
+      if (!strcmpstart(s, "Content-Encoding: ")) {
+        enc = s+18; break;
+      });
+
+    if (enc == NULL)
+      *compression = NO_METHOD;
+    else {
+      *compression = compression_method_get_by_name(enc);
+
+      if (*compression == UNKNOWN_METHOD)
+        log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
+                 escaped(enc));
+    }
+  }
+  SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
+  smartlist_free(parsed_headers);
+
+  return 0;
+}
+
+/** Return true iff <b>body</b> doesn't start with a plausible router or
+ * network-status or microdescriptor opening.  This is a sign of possible
+ * compression. */
+static int
+body_is_plausible(const char *body, size_t len, int purpose)
+{
+  int i;
+  if (len == 0)
+    return 1; /* empty bodies don't need decompression */
+  if (len < 32)
+    return 0;
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    return (!strcmpstart(body,"onion-key"));
+  }
+
+  if (!strcmpstart(body,"router") ||
+      !strcmpstart(body,"network-status"))
+    return 1;
+  for (i=0;i<32;++i) {
+    if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
+      return 0;
+  }
+
+  return 1;
+}
+
+/** Called when we've just fetched a bunch of router descriptors in
+ * <b>body</b>.  The list <b>which</b>, if present, holds digests for
+ * descriptors we requested: descriptor digests if <b>descriptor_digests</b>
+ * is true, or identity digests otherwise.  Parse the descriptors, validate
+ * them, and annotate them as having purpose <b>purpose</b> and as having been
+ * downloaded from <b>source</b>.
+ *
+ * Return the number of routers actually added. */
+static int
+load_downloaded_routers(const char *body, smartlist_t *which,
+                        int descriptor_digests,
+                        int router_purpose,
+                        const char *source)
+{
+  char buf[256];
+  char time_buf[ISO_TIME_LEN+1];
+  int added = 0;
+  int general = router_purpose == ROUTER_PURPOSE_GENERAL;
+  format_iso_time(time_buf, time(NULL));
+  tor_assert(source);
+
+  if (tor_snprintf(buf, sizeof(buf),
+                   "@downloaded-at %s\n"
+                   "@source %s\n"
+                   "%s%s%s", time_buf, escaped(source),
+                   !general ? "@purpose " : "",
+                   !general ? router_purpose_to_string(router_purpose) : "",
+                   !general ? "\n" : "")<0)
+    return added;
+
+  added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
+                                  descriptor_digests, buf);
+  if (added && general)
+    control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
+                            count_loading_descriptors_progress());
+  return added;
+}
+
+static int handle_response_fetch_certificate(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_status_vote(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_detached_signatures(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_desc(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_upload_dir(dir_connection_t *,
+                                      const response_handler_args_t *);
+static int handle_response_upload_vote(dir_connection_t *,
+                                       const response_handler_args_t *);
+static int handle_response_upload_signatures(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_renddesc_v2(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_upload_renddesc_v2(dir_connection_t *,
+                                              const response_handler_args_t *);
+static int handle_response_upload_hsdesc(dir_connection_t *,
+                                         const response_handler_args_t *);
+
+static int
+dir_client_decompress_response_body(char **bodyp, size_t *bodylenp,
+                                    dir_connection_t *conn,
+                                    compress_method_t compression,
+                                    int anonymized_connection)
+{
+  int rv = 0;
+  const char *body = *bodyp;
+  size_t body_len = *bodylenp;
+  int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+
+  int plausible = body_is_plausible(body, body_len, conn->base_.purpose);
+
+  if (plausible && compression == NO_METHOD) {
+    return 0;
+  }
+
+  int severity = LOG_DEBUG;
+  char *new_body = NULL;
+  size_t new_len = 0;
+  const char *description1, *description2;
+  int want_to_try_both = 0;
+  int tried_both = 0;
+  compress_method_t guessed = detect_compression_method(body, body_len);
+
+  description1 = compression_method_get_human_name(compression);
+
+  if (BUG(description1 == NULL))
+    description1 = compression_method_get_human_name(UNKNOWN_METHOD);
+
+  if (guessed == UNKNOWN_METHOD && !plausible)
+    description2 = "confusing binary junk";
+  else
+    description2 = compression_method_get_human_name(guessed);
+
+  /* Tell the user if we don't believe what we're told about compression.*/
+  want_to_try_both = (compression == UNKNOWN_METHOD ||
+                      guessed != compression);
+  if (want_to_try_both) {
+    severity = LOG_PROTOCOL_WARN;
+  }
+
+  tor_log(severity, LD_HTTP,
+          "HTTP body from server '%s:%d' was labeled as %s, "
+          "%s it seems to be %s.%s",
+          conn->base_.address, conn->base_.port, description1,
+          guessed != compression?"but":"and",
+          description2,
+          (compression>0 && guessed>0 && want_to_try_both)?
+          "  Trying both.":"");
+
+  /* Try declared compression first if we can.
+   * tor_compress_supports_method() also returns true for NO_METHOD.
+   * Ensure that the server is not sending us data compressed using a
+   * compression method that is not allowed for anonymous connections. */
+  if (anonymized_connection &&
+      ! allowed_anonymous_connection_compression_method(compression)) {
+    warn_disallowed_anonymous_compression_method(compression);
+    rv = -1;
+    goto done;
+  }
+
+  if (tor_compress_supports_method(compression)) {
+    tor_uncompress(&new_body, &new_len, body, body_len, compression,
+                   !allow_partial, LOG_PROTOCOL_WARN);
+    if (new_body) {
+      /* We succeeded with the declared compression method. Great! */
+      rv = 0;
+      goto done;
+    }
+  }
+
+  /* Okay, if that didn't work, and we think that it was compressed
+   * differently, try that. */
+  if (anonymized_connection &&
+      ! allowed_anonymous_connection_compression_method(guessed)) {
+    warn_disallowed_anonymous_compression_method(guessed);
+    rv = -1;
+    goto done;
+  }
+
+  if (tor_compress_supports_method(guessed) &&
+      compression != guessed) {
+    tor_uncompress(&new_body, &new_len, body, body_len, guessed,
+                   !allow_partial, LOG_INFO);
+    tried_both = 1;
+  }
+  /* If we're pretty sure that we have a compressed directory, and
+   * we didn't manage to uncompress it, then warn and bail. */
+  if (!plausible && !new_body) {
+    log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
+           "Unable to decompress HTTP body (tried %s%s%s, server '%s:%d').",
+           description1,
+           tried_both?" and ":"",
+           tried_both?description2:"",
+           conn->base_.address, conn->base_.port);
+    rv = -1;
+    goto done;
+  }
+
+ done:
+  if (new_body) {
+    if (rv == 0) {
+      /* success! */
+      tor_free(*bodyp);
+      *bodyp = new_body;
+      *bodylenp = new_len;
+    } else {
+      tor_free(new_body);
+    }
+  }
+
+  return rv;
+}
+
+/** We are a client, and we've finished reading the server's
+ * response. Parse it and act appropriately.
+ *
+ * If we're still happy with using this directory server in the future, return
+ * 0. Otherwise return -1; and the caller should consider trying the request
+ * again.
+ *
+ * The caller will take care of marking the connection for close.
+ */
+static int
+connection_dir_client_reached_eof(dir_connection_t *conn)
+{
+  char *body = NULL;
+  char *headers = NULL;
+  char *reason = NULL;
+  size_t body_len = 0;
+  int status_code;
+  time_t date_header = 0;
+  long apparent_skew;
+  compress_method_t compression;
+  int skewed = 0;
+  int rv;
+  int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+  size_t received_bytes;
+  const int anonymized_connection =
+    purpose_needs_anonymity(conn->base_.purpose,
+                            conn->router_purpose,
+                            conn->requested_resource);
+
+  received_bytes = connection_get_inbuf_len(TO_CONN(conn));
+
+  switch (connection_fetch_from_buf_http(TO_CONN(conn),
+                              &headers, MAX_HEADERS_SIZE,
+                              &body, &body_len, MAX_DIR_DL_SIZE,
+                              allow_partial)) {
+    case -1: /* overflow */
+      log_warn(LD_PROTOCOL,
+               "'fetch' response too large (server '%s:%d'). Closing.",
+               conn->base_.address, conn->base_.port);
+      return -1;
+    case 0:
+      log_info(LD_HTTP,
+               "'fetch' response not all here, but we're at eof. Closing.");
+      return -1;
+    /* case 1, fall through */
+  }
+
+  if (parse_http_response(headers, &status_code, &date_header,
+                          &compression, &reason) < 0) {
+    log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.",
+             conn->base_.address, conn->base_.port);
+
+    rv = -1;
+    goto done;
+  }
+  if (!reason) reason = tor_strdup("[no reason given]");
+
+  tor_log(LOG_DEBUG, LD_DIR,
+            "Received response from directory server '%s:%d': %d %s "
+            "(purpose: %d, response size: %"TOR_PRIuSZ
+#ifdef MEASUREMENTS_21206
+            ", data cells received: %d, data cells sent: %d"
+#endif
+            ", compression: %d)",
+            conn->base_.address, conn->base_.port, status_code,
+            escaped(reason), conn->base_.purpose,
+            (received_bytes),
+#ifdef MEASUREMENTS_21206
+            conn->data_cells_received, conn->data_cells_sent,
+#endif
+            compression);
+
+  if (conn->guard_state) {
+    /* we count the connection as successful once we can read from it.  We do
+     * not, however, delay use of the circuit here, since it's just for a
+     * one-hop directory request. */
+    /* XXXXprop271 note that this will not do the right thing for other
+     * waiting circuits that would be triggered by this circuit becoming
+     * complete/usable. But that's ok, I think.
+     */
+    entry_guard_succeeded(&conn->guard_state);
+    circuit_guard_state_free(conn->guard_state);
+    conn->guard_state = NULL;
+  }
+
+  /* now check if it's got any hints for us about our IP address. */
+  if (conn->dirconn_direct) {
+    char *guess = http_get_header(headers, X_ADDRESS_HEADER);
+    if (guess) {
+      router_new_address_suggestion(guess, conn);
+      tor_free(guess);
+    }
+  }
+
+  if (date_header > 0) {
+    /* The date header was written very soon after we sent our request,
+     * so compute the skew as the difference between sending the request
+     * and the date header.  (We used to check now-date_header, but that's
+     * inaccurate if we spend a lot of time downloading.)
+     */
+    apparent_skew = conn->base_.timestamp_last_write_allowed - date_header;
+    if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) {
+      int trusted = router_digest_is_trusted_dir(conn->identity_digest);
+      clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP,
+                         "directory", "DIRSERV");
+      skewed = 1; /* don't check the recommended-versions line */
+    } else {
+      log_debug(LD_HTTP, "Time on received directory is within tolerance; "
+                "we are %ld seconds skewed.  (That's okay.)", apparent_skew);
+    }
+  }
+  (void) skewed; /* skewed isn't used yet. */
+
+  if (status_code == 503) {
+    routerstatus_t *rs;
+    dir_server_t *ds;
+    const char *id_digest = conn->identity_digest;
+    log_info(LD_DIR,"Received http status code %d (%s) from server "
+             "'%s:%d'. I'll try again soon.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    time_t now = approx_time();
+    if ((rs = router_get_mutable_consensus_status_by_id(id_digest)))
+      rs->last_dir_503_at = now;
+    if ((ds = router_get_fallback_dirserver_by_digest(id_digest)))
+      ds->fake_status.last_dir_503_at = now;
+
+    rv = -1;
+    goto done;
+  }
+
+  if (dir_client_decompress_response_body(&body, &body_len,
+                             conn, compression, anonymized_connection) < 0) {
+    rv = -1;
+    goto done;
+  }
+
+  response_handler_args_t args;
+  memset(&args, 0, sizeof(args));
+  args.status_code = status_code;
+  args.reason = reason;
+  args.body = body;
+  args.body_len = body_len;
+  args.headers = headers;
+
+  switch (conn->base_.purpose) {
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      rv = handle_response_fetch_consensus(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      rv = handle_response_fetch_certificate(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+      rv = handle_response_fetch_status_vote(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+      rv = handle_response_fetch_detached_signatures(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      rv = handle_response_fetch_desc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      rv = handle_response_fetch_microdesc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+      rv = handle_response_fetch_renddesc_v2(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_DIR:
+      rv = handle_response_upload_dir(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+      rv = handle_response_upload_signatures(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_VOTE:
+      rv = handle_response_upload_vote(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+      rv = handle_response_upload_renddesc_v2(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      rv = handle_response_upload_hsdesc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_HSDESC:
+      rv = handle_response_fetch_hsdesc_v3(conn, &args);
+      break;
+    default:
+      tor_assert_nonfatal_unreached();
+      rv = -1;
+      break;
+  }
+
+ done:
+  tor_free(body);
+  tor_free(headers);
+  tor_free(reason);
+  return rv;
+}
+
+/**
+ * Handler function: processes a response to a request for a networkstatus
+ * consensus document by checking the consensus, storing it, and marking
+ * router requests as reachable.
+ **/
+STATIC int
+handle_response_fetch_consensus(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS);
+  const int status_code = args->status_code;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+  const char *reason = args->reason;
+  const time_t now = approx_time();
+
+  const char *consensus;
+  char *new_consensus = NULL;
+  const char *sourcename;
+
+  int r;
+  const char *flavname = conn->requested_resource;
+  if (status_code != 200) {
+    int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
+    tor_log(severity, LD_DIR,
+            "Received http status code %d (%s) from server "
+            "'%s:%d' while fetching consensus directory.",
+            status_code, escaped(reason), conn->base_.address,
+            conn->base_.port);
+    networkstatus_consensus_download_failed(status_code, flavname);
+    return -1;
+  }
+
+  if (looks_like_a_consensus_diff(body, body_len)) {
+    /* First find our previous consensus. Maybe it's in ram, maybe not. */
+    cached_dir_t *cd = dirserv_get_consensus(flavname);
+    const char *consensus_body;
+    char *owned_consensus = NULL;
+    if (cd) {
+      consensus_body = cd->dir;
+    } else {
+      owned_consensus = networkstatus_read_cached_consensus(flavname);
+      consensus_body = owned_consensus;
+    }
+    if (!consensus_body) {
+      log_warn(LD_DIR, "Received a consensus diff, but we can't find "
+               "any %s-flavored consensus in our current cache.",flavname);
+      networkstatus_consensus_download_failed(0, flavname);
+      // XXXX if this happens too much, see below
+      return -1;
+    }
+
+    new_consensus = consensus_diff_apply(consensus_body, body);
+    tor_free(owned_consensus);
+    if (new_consensus == NULL) {
+      log_warn(LD_DIR, "Could not apply consensus diff received from server "
+               "'%s:%d'", conn->base_.address, conn->base_.port);
+      // XXXX If this happens too many times, we should maybe not use
+      // XXXX this directory for diffs any more?
+      networkstatus_consensus_download_failed(0, flavname);
+      return -1;
+    }
+    log_info(LD_DIR, "Applied consensus diff (size %d) from server "
+             "'%s:%d', resulting in a new consensus document (size %d).",
+             (int)body_len, conn->base_.address, conn->base_.port,
+             (int)strlen(new_consensus));
+    consensus = new_consensus;
+    sourcename = "generated based on a diff";
+  } else {
+    log_info(LD_DIR,"Received consensus directory (body size %d) from server "
+             "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
+    consensus = body;
+    sourcename = "downloaded";
+  }
+
+  if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+                                             conn->identity_digest))<0) {
+    log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
+           "Unable to load %s consensus directory %s from "
+           "server '%s:%d'. I'll try again soon.",
+           flavname, sourcename, conn->base_.address, conn->base_.port);
+    networkstatus_consensus_download_failed(0, flavname);
+    tor_free(new_consensus);
+    return -1;
+  }
+
+  /* If we launched other fetches for this consensus, cancel them. */
+  connection_dir_close_consensus_fetches(conn, flavname);
+
+  /* update the list of routers and directory guards */
+  routers_update_all_from_networkstatus(now, 3);
+  update_microdescs_from_networkstatus(now);
+  directory_info_has_arrived(now, 0, 0);
+
+  if (authdir_mode_v3(get_options())) {
+    sr_act_post_consensus(
+                     networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
+  }
+  log_info(LD_DIR, "Successfully loaded consensus.");
+
+  tor_free(new_consensus);
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for one or more
+ * authority certificates
+ **/
+static int
+handle_response_fetch_certificate(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  if (status_code != 200) {
+    log_warn(LD_DIR,
+             "Received http status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/keys/%s\".",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port, conn->requested_resource);
+    connection_dir_download_cert_failed(conn, status_code);
+    return -1;
+  }
+  log_info(LD_DIR,"Received authority certificates (body size %d) from "
+           "server '%s:%d'",
+           (int)body_len, conn->base_.address, conn->base_.port);
+
+  /*
+   * Tell trusted_dirs_load_certs_from_string() whether it was by fp
+   * or fp-sk pair.
+   */
+  int src_code = -1;
+  if (!strcmpstart(conn->requested_resource, "fp/")) {
+    src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST;
+  } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) {
+    src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST;
+  }
+
+  if (src_code != -1) {
+    if (trusted_dirs_load_certs_from_string(body, src_code, 1,
+                                            conn->identity_digest)<0) {
+      log_warn(LD_DIR, "Unable to parse fetched certificates");
+      /* if we fetched more than one and only some failed, the successful
+       * ones got flushed to disk so it's safe to call this on them */
+      connection_dir_download_cert_failed(conn, status_code);
+    } else {
+      time_t now = approx_time();
+      directory_info_has_arrived(now, 0, 0);
+      log_info(LD_DIR, "Successfully loaded certificates from fetch.");
+    }
+  } else {
+    log_warn(LD_DIR,
+             "Couldn't figure out what to do with fetched certificates for "
+             "unknown resource %s",
+             conn->requested_resource);
+    connection_dir_download_cert_failed(conn, status_code);
+  }
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for an authority's
+ * current networkstatus vote.
+ **/
+static int
+handle_response_fetch_status_vote(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  const char *msg;
+  int st;
+  log_info(LD_DIR,"Got votes (body size %d) from server %s:%d",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (status_code != 200) {
+    log_warn(LD_DIR,
+             "Received http status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port, conn->requested_resource);
+    return -1;
+  }
+  dirvote_add_vote(body, &msg, &st);
+  if (st > 299) {
+    log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
+  } else {
+    log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for the signatures
+ * that an authority knows about on a given consensus.
+ **/
+static int
+handle_response_fetch_detached_signatures(dir_connection_t *conn,
+                                          const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  const char *msg = NULL;
+  log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (status_code != 200) {
+    log_warn(LD_DIR,
+        "Received http status code %d (%s) from server '%s:%d' while fetching "
+        "\"/tor/status-vote/next/consensus-signatures.z\".",
+        status_code, escaped(reason), conn->base_.address,
+        conn->base_.port);
+    return -1;
+  }
+  if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) {
+    log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
+             conn->base_.address, conn->base_.port, msg?msg:"???");
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a group of server
+ * descriptors or an extrainfo documents.
+ **/
+static int
+handle_response_fetch_desc(dir_connection_t *conn,
+                           const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
+  smartlist_t *which = NULL;
+  int n_asked_for = 0;
+  int descriptor_digests = conn->requested_resource &&
+    !strcmpstart(conn->requested_resource,"d/");
+  log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'",
+           was_ei ? "extra server info" : "server info",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (conn->requested_resource &&
+      (!strcmpstart(conn->requested_resource,"d/") ||
+       !strcmpstart(conn->requested_resource,"fp/"))) {
+    which = smartlist_new();
+    dir_split_resource_into_fingerprints(conn->requested_resource +
+                                         (descriptor_digests ? 2 : 3),
+                                         which, NULL, 0);
+    n_asked_for = smartlist_len(which);
+  }
+  if (status_code != 200) {
+    int dir_okay = status_code == 404 ||
+      (status_code == 400 && !strcmp(reason, "Servers unavailable."));
+    /* 404 means that it didn't have them; no big deal.
+     * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
+    log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
+           "Received http status code %d (%s) from server '%s:%d' "
+           "while fetching \"/tor/server/%s\". I'll try again soon.",
+           status_code, escaped(reason), conn->base_.address,
+           conn->base_.port, conn->requested_resource);
+    if (!which) {
+      connection_dir_download_routerdesc_failed(conn);
+    } else {
+      dir_routerdesc_download_failed(which, status_code,
+                                     conn->router_purpose,
+                                     was_ei, descriptor_digests);
+      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+      smartlist_free(which);
+    }
+    return dir_okay ? 0 : -1;
+  }
+  /* Learn the routers, assuming we requested by fingerprint or "all"
+   * or "authority".
+   *
+   * We use "authority" to fetch our own descriptor for
+   * testing, and to fetch bridge descriptors for bootstrapping. Ignore
+   * the output of "authority" requests unless we are using bridges,
+   * since otherwise they'll be the response from reachability tests,
+   * and we don't really want to add that to our routerlist. */
+  if (which || (conn->requested_resource &&
+                (!strcmpstart(conn->requested_resource, "all") ||
+                 (!strcmpstart(conn->requested_resource, "authority") &&
+                  get_options()->UseBridges)))) {
+    /* as we learn from them, we remove them from 'which' */
+    if (was_ei) {
+      router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
+                                        descriptor_digests);
+    } else {
+      //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
+      //                       descriptor_digests, conn->router_purpose);
+      if (load_downloaded_routers(body, which, descriptor_digests,
+                                  conn->router_purpose,
+                                  conn->base_.address)) {
+        time_t now = approx_time();
+        directory_info_has_arrived(now, 0, 1);
+      }
+    }
+  }
+  if (which) { /* mark remaining ones as failed */
+    log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
+             n_asked_for-smartlist_len(which), n_asked_for,
+             was_ei ? "extra-info documents" : "router descriptors",
+             conn->base_.address, (int)conn->base_.port);
+    if (smartlist_len(which)) {
+      dir_routerdesc_download_failed(which, status_code,
+                                     conn->router_purpose,
+                                     was_ei, descriptor_digests);
+    }
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+  }
+  if (directory_conn_is_self_reachability_test(conn))
+    router_dirport_found_reachable();
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a group of
+ * microdescriptors
+ **/
+STATIC int
+handle_response_fetch_microdesc(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  smartlist_t *which = NULL;
+  log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
+           "body size %d) from server '%s:%d'",
+           status_code, (int)body_len, conn->base_.address,
+           conn->base_.port);
+  tor_assert(conn->requested_resource &&
+             !strcmpstart(conn->requested_resource, "d/"));
+  tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
+  which = smartlist_new();
+  dir_split_resource_into_fingerprints(conn->requested_resource+2,
+                                       which, NULL,
+                                       DSR_DIGEST256|DSR_BASE64);
+  if (status_code != 200) {
+    log_info(LD_DIR, "Received status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
+             "soon.",
+             status_code, escaped(reason), conn->base_.address,
+             (int)conn->base_.port, conn->requested_resource);
+    dir_microdesc_download_failed(which, status_code, conn->identity_digest);
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+    return 0;
+  } else {
+    smartlist_t *mds;
+    time_t now = approx_time();
+    mds = microdescs_add_to_cache(get_microdesc_cache(),
+                                  body, body+body_len, SAVED_NOWHERE, 0,
+                                  now, which);
+    if (smartlist_len(which)) {
+      /* Mark remaining ones as failed. */
+      dir_microdesc_download_failed(which, status_code, conn->identity_digest);
+    }
+    if (mds && smartlist_len(mds)) {
+      control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
+                              count_loading_descriptors_progress());
+      directory_info_has_arrived(now, 0, 1);
+    }
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+    smartlist_free(mds);
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload our
+ * router descriptor.
+ **/
+static int
+handle_response_upload_dir(dir_connection_t *conn,
+                           const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *headers = args->headers;
+
+  switch (status_code) {
+  case 200: {
+    dir_server_t *ds =
+      router_get_trusteddirserver_by_digest(conn->identity_digest);
+    char *rejected_hdr = http_get_header(headers,
+                                         "X-Descriptor-Not-New: ");
+    if (rejected_hdr) {
+      if (!strcmp(rejected_hdr, "Yes")) {
+        log_info(LD_GENERAL,
+                 "Authority '%s' declined our descriptor (not new)",
+                 ds->nickname);
+        /* XXXX use this information; be sure to upload next one
+         * sooner. -NM */
+        /* XXXX++ On further thought, the task above implies that we're
+         * basing our regenerate-descriptor time on when we uploaded the
+         * last descriptor, not on the published time of the last
+         * descriptor.  If those are different, that's a bad thing to
+         * do. -NM */
+      }
+      tor_free(rejected_hdr);
+    }
+    log_info(LD_GENERAL,"eof (status 200) after uploading server "
+             "descriptor: finished.");
+    control_event_server_status(
+                   LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
+                   conn->base_.address, conn->base_.port);
+
+    ds->has_accepted_serverdesc = 1;
+    if (directories_have_accepted_server_descriptor())
+      control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
+  }
+    break;
+  case 400:
+    log_warn(LD_GENERAL,"http status 400 (%s) response from "
+             "dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    control_event_server_status(LOG_WARN,
+                    "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
+                    conn->base_.address, conn->base_.port, escaped(reason));
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "descriptor to server '%s:%d'. Possibly the server is "
+             "misconfigured?",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to POST request to upload our
+ * own networkstatus vote.
+ **/
+static int
+handle_response_upload_vote(dir_connection_t *conn,
+                            const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  switch (status_code) {
+  case 200: {
+    log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
+               conn->base_.address, conn->base_.port);
+  }
+    break;
+  case 400:
+    log_warn(LD_DIR,"http status 400 (%s) response after uploading "
+             "vote to dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "vote to server '%s:%d'.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to POST request to upload our
+ * view of the signatures on the current consensus.
+ **/
+static int
+handle_response_upload_signatures(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  switch (status_code) {
+  case 200: {
+    log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
+               conn->base_.address, conn->base_.port);
+  }
+    break;
+  case 400:
+    log_warn(LD_DIR,"http status 400 (%s) response after uploading "
+             "signatures to dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "signatures to server '%s:%d'.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a v3 hidden service
+ * descriptor.
+ **/
+STATIC int
+handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  tor_assert(conn->hs_ident);
+
+  log_info(LD_REND,"Received v3 hsdesc (body size %d, status %d (%s))",
+           (int)body_len, status_code, escaped(reason));
+
+  switch (status_code) {
+  case 200:
+    /* We got something: Try storing it in the cache. */
+    if (hs_cache_store_as_client(body, &conn->hs_ident->identity_pk) < 0) {
+      log_warn(LD_REND, "Failed to store hidden service descriptor");
+      /* Fire control port FAILED event. */
+      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                   "BAD_DESC");
+      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                    NULL);
+    } else {
+      log_info(LD_REND, "Stored hidden service descriptor successfully.");
+      TO_CONN(conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC;
+      hs_client_desc_has_arrived(conn->hs_ident);
+      /* Fire control port RECEIVED event. */
+      hs_control_desc_event_received(conn->hs_ident, conn->identity_digest);
+      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                    body);
+    }
+    break;
+  case 404:
+    /* Not there. We'll retry when connection_about_to_close_connection()
+     * tries to clean this conn up. */
+    log_info(LD_REND, "Fetching hidden service v3 descriptor not found: "
+                      "Retrying at another directory.");
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "NOT_FOUND");
+    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                  NULL);
+    break;
+  case 400:
+    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+                      "http status 400 (%s). Dirserver didn't like our "
+                      "query? Retrying at another directory.",
+             escaped(reason));
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "QUERY_REJECTED");
+    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                  NULL);
+    break;
+  default:
+    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+             "http status %d (%s) response unexpected from HSDir server "
+             "'%s:%d'. Retrying at another directory.",
+             status_code, escaped(reason), TO_CONN(conn)->address,
+             TO_CONN(conn)->port);
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "UNEXPECTED");
+    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                  NULL);
+    break;
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a v2 hidden service
+ * descriptor.
+ **/
+static int
+handle_response_fetch_renddesc_v2(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+#define SEND_HS_DESC_FAILED_EVENT(reason)                               \
+  (control_event_hsv2_descriptor_failed(conn->rend_data,                \
+                                        conn->identity_digest,          \
+                                        reason))
+#define SEND_HS_DESC_FAILED_CONTENT()                                   \
+  (control_event_hs_descriptor_content(                                 \
+                                rend_data_get_address(conn->rend_data), \
+                                conn->requested_resource,               \
+                                conn->identity_digest,                  \
+                                NULL))
+
+  tor_assert(conn->rend_data);
+  log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d "
+           "(%s))",
+           (int)body_len, status_code, escaped(reason));
+  switch (status_code) {
+  case 200:
+    {
+      rend_cache_entry_t *entry = NULL;
+
+      if (rend_cache_store_v2_desc_as_client(body,
+                                             conn->requested_resource,
+                                             conn->rend_data, &entry) < 0) {
+        log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
+                 "Retrying at another directory.");
+        /* We'll retry when connection_about_to_close_connection()
+         * cleans this dir conn up. */
+        SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
+        SEND_HS_DESC_FAILED_CONTENT();
+      } else {
+        char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
+        /* Should never be NULL here if we found the descriptor. */
+        tor_assert(entry);
+        rend_get_service_id(entry->parsed->pk, service_id);
+
+        /* success. notify pending connections about this. */
+        log_info(LD_REND, "Successfully fetched v2 rendezvous "
+                 "descriptor.");
+        control_event_hsv2_descriptor_received(service_id,
+                                               conn->rend_data,
+                                               conn->identity_digest);
+        control_event_hs_descriptor_content(service_id,
+                                            conn->requested_resource,
+                                            conn->identity_digest,
+                                            body);
+        conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2;
+        rend_client_desc_trynow(service_id);
+        memwipe(service_id, 0, sizeof(service_id));
+      }
+      break;
+    }
+  case 404:
+    /* Not there. We'll retry when
+     * connection_about_to_close_connection() cleans this conn up. */
+    log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
+             "Retrying at another directory.");
+    SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  case 400:
+    log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
+             "http status 400 (%s). Dirserver didn't like our "
+             "v2 rendezvous query? Retrying at another directory.",
+             escaped(reason));
+    SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  default:
+    log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
+             "http status %d (%s) response unexpected while "
+             "fetching v2 hidden service descriptor (server '%s:%d'). "
+             "Retrying at another directory.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload a v2
+ * hidden service descriptor.
+ **/
+static int
+handle_response_upload_renddesc_v2(dir_connection_t *conn,
+                                   const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+#define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason)                        \
+    (control_event_hs_descriptor_upload_failed(                         \
+                                conn->identity_digest,                  \
+                                rend_data_get_address(conn->rend_data), \
+                                reason))
+
+  log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
+           "(%s))",
+           status_code, escaped(reason));
+  /* Without the rend data, we'll have a problem identifying what has been
+   * uploaded for which service. */
+  tor_assert(conn->rend_data);
+  switch (status_code) {
+  case 200:
+    log_info(LD_REND,
+             "Uploading rendezvous descriptor: finished with status "
+             "200 (%s)", escaped(reason));
+    control_event_hs_descriptor_uploaded(conn->identity_digest,
+                                   rend_data_get_address(conn->rend_data));
+    rend_service_desc_has_uploaded(conn->rend_data);
+    break;
+  case 400:
+    log_warn(LD_REND,"http status 400 (%s) response from dirserver "
+             "'%s:%d'. Malformed rendezvous descriptor?",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
+    break;
+  default:
+    log_warn(LD_REND,"http status %d (%s) response unexpected (server "
+             "'%s:%d').",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
+    break;
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload an
+ * hidden service descriptor.
+ **/
+static int
+handle_response_upload_hsdesc(dir_connection_t *conn,
+                              const response_handler_args_t *args)
+{
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  tor_assert(conn);
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_HSDESC);
+
+  log_info(LD_REND, "Uploaded hidden service descriptor (status %d "
+                    "(%s))",
+           status_code, escaped(reason));
+  /* For this directory response, it MUST have an hidden service identifier on
+   * this connection. */
+  tor_assert(conn->hs_ident);
+  switch (status_code) {
+  case 200:
+    log_info(LD_REND, "Uploading hidden service descriptor: "
+                      "finished with status 200 (%s)", escaped(reason));
+    hs_control_desc_event_uploaded(conn->hs_ident, conn->identity_digest);
+    break;
+  case 400:
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Uploading hidden service descriptor: http "
+           "status 400 (%s) response from dirserver "
+           "'%s:%d'. Malformed hidden service descriptor?",
+           escaped(reason), conn->base_.address, conn->base_.port);
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "UPLOAD_REJECTED");
+    break;
+  default:
+    log_warn(LD_REND, "Uploading hidden service descriptor: http "
+                      "status %d (%s) response unexpected (server "
+                      "'%s:%d').",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "UNEXPECTED");
+    break;
+  }
+
+  return 0;
+}
+
+/** Called when a directory connection reaches EOF. */
+int
+connection_dir_reached_eof(dir_connection_t *conn)
+{
+  int retval;
+  if (conn->base_.state != DIR_CONN_STATE_CLIENT_READING) {
+    log_info(LD_HTTP,"conn reached eof, not reading. [state=%d] Closing.",
+             conn->base_.state);
+    connection_close_immediate(TO_CONN(conn)); /* error: give up on flushing */
+    connection_mark_for_close(TO_CONN(conn));
+    return -1;
+  }
+
+  retval = connection_dir_client_reached_eof(conn);
+  if (retval == 0) /* success */
+    conn->base_.state = DIR_CONN_STATE_CLIENT_FINISHED;
+  connection_mark_for_close(TO_CONN(conn));
+  return retval;
+}
+
+/** If any directory object is arriving, and it's over 10MB large, we're
+ * getting DoS'd.  (As of 0.1.2.x, raw directories are about 1MB, and we never
+ * ask for more than 96 router descriptors at a time.)
+ */
+#define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
+
+#define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5)
+
+/** Read handler for directory connections.  (That's connections <em>to</em>
+ * directory servers and connections <em>at</em> directory servers.)
+ */
+int
+connection_dir_process_inbuf(dir_connection_t *conn)
+{
+  size_t max_size;
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  /* Directory clients write, then read data until they receive EOF;
+   * directory servers read data until they get an HTTP command, then
+   * write their response (when it's finished flushing, they mark for
+   * close).
+   */
+
+  /* If we're on the dirserver side, look for a command. */
+  if (conn->base_.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) {
+    if (directory_handle_command(conn) < 0) {
+      connection_mark_for_close(TO_CONN(conn));
+      return -1;
+    }
+    return 0;
+  }
+
+  max_size =
+    (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ?
+    MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE;
+
+  if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) {
+    log_warn(LD_HTTP,
+             "Too much data received from directory connection (%s): "
+             "denial of service attempt, or you need to upgrade?",
+             conn->base_.address);
+    connection_mark_for_close(TO_CONN(conn));
+    return -1;
+  }
+
+  if (!conn->base_.inbuf_reached_eof)
+    log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf.");
+  return 0;
+}
+
+/** We are closing a dir connection: If <b>dir_conn</b> is a dir connection
+ *  that tried to fetch an HS descriptor, check if it successfully fetched it,
+ *  or if we need to try again. */
+static void
+refetch_hsdesc_if_needed(dir_connection_t *dir_conn)
+{
+  connection_t *conn = TO_CONN(dir_conn);
+
+  /* If we were trying to fetch a v2 rend desc and did not succeed, retry as
+   * needed. (If a fetch is successful, the connection state is changed to
+   * DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 or DIR_PURPOSE_HAS_FETCHED_HSDESC to
+   * mark that refetching is unnecessary.) */
+  if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 &&
+      dir_conn->rend_data &&
+      rend_valid_v2_service_id(
+           rend_data_get_address(dir_conn->rend_data))) {
+    rend_client_refetch_v2_renddesc(dir_conn->rend_data);
+  }
+
+  /* Check for v3 rend desc fetch */
+  if (conn->purpose == DIR_PURPOSE_FETCH_HSDESC &&
+      dir_conn->hs_ident &&
+      !ed25519_public_key_is_zero(&dir_conn->hs_ident->identity_pk)) {
+    hs_client_refetch_hsdesc(&dir_conn->hs_ident->identity_pk);
+  }
+}
+
+/** Called when we're about to finally unlink and free a directory connection:
+ * perform necessary accounting and cleanup */
+void
+connection_dir_about_to_close(dir_connection_t *dir_conn)
+{
+  connection_t *conn = TO_CONN(dir_conn);
+
+  if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) {
+    /* It's a directory connection and connecting or fetching
+     * failed: forget about this router, and maybe try again. */
+    connection_dir_request_failed(dir_conn);
+  }
+
+  refetch_hsdesc_if_needed(dir_conn);
+}
+
+/** Create an http response for the client <b>conn</b> out of
+ * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
+ */
+static void
+write_short_http_response(dir_connection_t *conn, int status,
+                       const char *reason_phrase)
+{
+  char *buf = NULL;
+  char *datestring = NULL;
+
+  IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
+    reason_phrase = "unspecified";
+  }
+
+  if (server_mode(get_options())) {
+    /* include the Date: header, but only if we're a relay or bridge */
+    char datebuf[RFC1123_TIME_LEN+1];
+    format_rfc1123_time(datebuf, time(NULL));
+    tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
+  }
+
+  tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
+               status, reason_phrase, datestring?datestring:"");
+
+  log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
+  connection_buf_add(buf, strlen(buf), TO_CONN(conn));
+
+  tor_free(datestring);
+  tor_free(buf);
+}
+
+/** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
+ * with <b>type</b> as the Content-Type.
+ *
+ * If <b>length</b> is nonnegative, it is the Content-Length.
+ * If <b>encoding</b> is provided, it is the Content-Encoding.
+ * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
+ * up to cache_lifetime seconds.  Otherwise, the content may not be cached. */
+static void
+write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
+                           const char *type, const char *encoding,
+                           const char *extra_headers,
+                           long cache_lifetime)
+{
+  char date[RFC1123_TIME_LEN+1];
+  time_t now = time(NULL);
+  buf_t *buf = buf_new_with_capacity(1024);
+
+  tor_assert(conn);
+
+  format_rfc1123_time(date, now);
+
+  buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
+  if (type) {
+    buf_add_printf(buf, "Content-Type: %s\r\n", type);
+  }
+  if (!is_local_addr(&conn->base_.addr)) {
+    /* Don't report the source address for a nearby/private connection.
+     * Otherwise we tend to mis-report in cases where incoming ports are
+     * being forwarded to a Tor server running behind the firewall. */
+    buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
+  }
+  if (encoding) {
+    buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
+  }
+  if (length >= 0) {
+    buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
+  }
+  if (cache_lifetime > 0) {
+    char expbuf[RFC1123_TIME_LEN+1];
+    format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
+    /* We could say 'Cache-control: max-age=%d' here if we start doing
+     * http/1.1 */
+    buf_add_printf(buf, "Expires: %s\r\n", expbuf);
+  } else if (cache_lifetime == 0) {
+    /* We could say 'Cache-control: no-cache' here if we start doing
+     * http/1.1 */
+    buf_add_string(buf, "Pragma: no-cache\r\n");
+  }
+  if (extra_headers) {
+    buf_add_string(buf, extra_headers);
+  }
+  buf_add_string(buf, "\r\n");
+
+  connection_buf_add_buf(TO_CONN(conn), buf);
+  buf_free(buf);
+}
+
+/** As write_http_response_header_impl, but sets encoding and content-typed
+ * based on whether the response will be <b>compressed</b> or not. */
+static void
+write_http_response_headers(dir_connection_t *conn, ssize_t length,
+                            compress_method_t method,
+                            const char *extra_headers, long cache_lifetime)
+{
+  const char *methodname = compression_method_get_name(method);
+  const char *doctype;
+  if (method == NO_METHOD)
+    doctype = "text/plain";
+  else
+    doctype = "application/octet-stream";
+  write_http_response_header_impl(conn, length,
+                                  doctype,
+                                  methodname,
+                                  extra_headers,
+                                  cache_lifetime);
+}
+
+/** As write_http_response_headers, but assumes extra_headers is NULL */
+static void
+write_http_response_header(dir_connection_t *conn, ssize_t length,
+                           compress_method_t method,
+                           long cache_lifetime)
+{
+  write_http_response_headers(conn, length, method, NULL, cache_lifetime);
+}
+
+/** Array of compression methods to use (if supported) for serving
+ * precompressed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_precompressed[] = {
+  LZMA_METHOD,
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Array of compression methods to use (if supported) for serving
+ * streamed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_streaming_compression[] = {
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Array of allowed compression methods to use (if supported) when receiving a
+ * response from a request that was required to be anonymous. */
+static compress_method_t client_meth_allowed_anonymous_compression[] = {
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
+ * and convert them to a bitfield where compression method x is supported if
+ * and only if 1 &lt;&lt; x is set in the bitfield. */
+STATIC unsigned
+parse_accept_encoding_header(const char *h)
+{
+  unsigned result = (1u << NO_METHOD);
+  smartlist_t *methods = smartlist_new();
+  smartlist_split_string(methods, h, ",",
+             SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+  SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
+    compress_method_t method = compression_method_get_by_name(m);
+    if (method != UNKNOWN_METHOD) {
+      tor_assert(((unsigned)method) < 8*sizeof(unsigned));
+      result |= (1u << method);
+    }
+  } SMARTLIST_FOREACH_END(m);
+  SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
+    tor_free(m);
+  } SMARTLIST_FOREACH_END(m);
+  smartlist_free(methods);
+  return result;
+}
+
+/** Array of compression methods to use (if supported) for requesting
+ * compressed data, ordered from best to worst. */
+static compress_method_t client_meth_pref[] = {
+  LZMA_METHOD,
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Return a newly allocated string containing a comma separated list of
+ * supported encodings. */
+STATIC char *
+accept_encoding_header(void)
+{
+  smartlist_t *methods = smartlist_new();
+  char *header = NULL;
+  compress_method_t method;
+  unsigned i;
+
+  for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) {
+    method = client_meth_pref[i];
+    if (tor_compress_supports_method(method))
+      smartlist_add(methods, (char *)compression_method_get_name(method));
+  }
+
+  header = smartlist_join_strings(methods, ", ", 0, NULL);
+  smartlist_free(methods);
+
+  return header;
+}
+
+/** Decide whether a client would accept the consensus we have.
+ *
+ * Clients can say they only want a consensus if it's signed by more
+ * than half the authorities in a list.  They pass this list in
+ * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
+ *
+ * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
+ * of the full authority identity digest. (Only strings of even length,
+ * i.e. encodings of full bytes, are handled correctly.  In the case
+ * of an odd number of hex digits the last one is silently ignored.)
+ *
+ * Returns 1 if more than half of the requested authorities signed the
+ * consensus, 0 otherwise.
+ */
+int
+client_likes_consensus(const struct consensus_cache_entry_t *ent,
+                       const char *want_url)
+{
+  smartlist_t *voters = smartlist_new();
+  int need_at_least;
+  int have = 0;
+
+  if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
+    smartlist_free(voters);
+    return 1; // We don't know the voters; assume the client won't mind. */
+  }
+
+  smartlist_t *want_authorities = smartlist_new();
+  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
+  need_at_least = smartlist_len(want_authorities)/2+1;
+
+  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
+
+    SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
+      if (!strcasecmpstart(digest, want_digest)) {
+        have++;
+        break;
+      };
+    } SMARTLIST_FOREACH_END(digest);
+
+    /* early exit, if we already have enough */
+    if (have >= need_at_least)
+      break;
+  } SMARTLIST_FOREACH_END(want_digest);
+
+  SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
+  smartlist_free(want_authorities);
+  SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
+  smartlist_free(voters);
+  return (have >= need_at_least);
+}
+
+/** Return the compression level we should use for sending a compressed
+ * response of size <b>n_bytes</b>. */
+STATIC compression_level_t
+choose_compression_level(ssize_t n_bytes)
+{
+  if (! have_been_under_memory_pressure()) {
+    return HIGH_COMPRESSION; /* we have plenty of RAM. */
+  } else if (n_bytes < 0) {
+    return HIGH_COMPRESSION; /* unknown; might be big. */
+  } else if (n_bytes < 1024) {
+    return LOW_COMPRESSION;
+  } else if (n_bytes < 2048) {
+    return MEDIUM_COMPRESSION;
+  } else {
+    return HIGH_COMPRESSION;
+  }
+}
+
+/** Information passed to handle a GET request. */
+typedef struct get_handler_args_t {
+  /** Bitmask of compression methods that the client said (or implied) it
+   * supported. */
+  unsigned compression_supported;
+  /** If nonzero, the time included an if-modified-since header with this
+   * value. */
+  time_t if_modified_since;
+  /** String containing the requested URL or resource. */
+  const char *url;
+  /** String containing the HTTP headers */
+  const char *headers;
+} get_handler_args_t;
+
+/** Entry for handling an HTTP GET request.
+ *
+ * This entry matches a request if "string" is equal to the requested
+ * resource, or if "is_prefix" is true and "string" is a prefix of the
+ * requested resource.
+ *
+ * The 'handler' function is called to handle the request.  It receives
+ * an arguments structure, and must return 0 on success or -1 if we should
+ * close the connection.
+ **/
+typedef struct url_table_ent_s {
+  const char *string;
+  int is_prefix;
+  int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
+} url_table_ent_t;
+
+static int handle_get_frontpage(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_current_consensus(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_status_vote(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_microdesc(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_descriptor(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_keys(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                                       const get_handler_args_t *args);
+static int handle_get_robots(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_networkstatus_bridges(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+
+/** Table for handling GET requests. */
+static const url_table_ent_t url_table[] = {
+  { "/tor/", 0, handle_get_frontpage },
+  { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
+  { "/tor/status-vote/current/", 1, handle_get_status_vote },
+  { "/tor/status-vote/next/", 1, handle_get_status_vote },
+  { "/tor/micro/d/", 1, handle_get_microdesc },
+  { "/tor/server/", 1, handle_get_descriptor },
+  { "/tor/extra/", 1, handle_get_descriptor },
+  { "/tor/keys/", 1, handle_get_keys },
+  { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
+  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
+  { "/tor/robots.txt", 0, handle_get_robots },
+  { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
+  { NULL, 0, NULL },
+};
+
+/** Helper function: called when a dirserver gets a complete HTTP GET
+ * request.  Look for a request for a directory or for a rendezvous
+ * service descriptor.  On finding one, write a response into
+ * conn-\>outbuf.  If the request is unrecognized, send a 404.
+ * Return 0 if we handled this successfully, or -1 if we need to close
+ * the connection. */
+MOCK_IMPL(STATIC int,
+directory_handle_command_get,(dir_connection_t *conn, const char *headers,
+                              const char *req_body, size_t req_body_len))
+{
+  char *url, *url_mem, *header;
+  time_t if_modified_since = 0;
+  int zlib_compressed_in_url;
+  unsigned compression_methods_supported;
+
+  /* We ignore the body of a GET request. */
+  (void)req_body;
+  (void)req_body_len;
+
+  log_debug(LD_DIRSERV,"Received GET command.");
+
+  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
+
+  if (parse_http_url(headers, &url) < 0) {
+    write_short_http_response(conn, 400, "Bad request");
+    return 0;
+  }
+  if ((header = http_get_header(headers, "If-Modified-Since: "))) {
+    struct tm tm;
+    if (parse_http_time(header, &tm) == 0) {
+      if (tor_timegm(&tm, &if_modified_since)<0) {
+        if_modified_since = 0;
+      } else {
+        log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
+      }
+    }
+    /* The correct behavior on a malformed If-Modified-Since header is to
+     * act as if no If-Modified-Since header had been given. */
+    tor_free(header);
+  }
+  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
+
+  url_mem = url;
+  {
+    size_t url_len = strlen(url);
+
+    zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
+    if (zlib_compressed_in_url) {
+      url[url_len-2] = '\0';
+    }
+  }
+
+  if ((header = http_get_header(headers, "Accept-Encoding: "))) {
+    compression_methods_supported = parse_accept_encoding_header(header);
+    tor_free(header);
+  } else {
+    compression_methods_supported = (1u << NO_METHOD);
+  }
+  if (zlib_compressed_in_url) {
+    compression_methods_supported |= (1u << ZLIB_METHOD);
+  }
+
+  /* Remove all methods that we don't both support. */
+  compression_methods_supported &= tor_compress_get_supported_method_bitmask();
+
+  get_handler_args_t args;
+  args.url = url;
+  args.headers = headers;
+  args.if_modified_since = if_modified_since;
+  args.compression_supported = compression_methods_supported;
+
+  int i, result = -1;
+  for (i = 0; url_table[i].string; ++i) {
+    int match;
+    if (url_table[i].is_prefix) {
+      match = !strcmpstart(url, url_table[i].string);
+    } else {
+      match = !strcmp(url, url_table[i].string);
+    }
+    if (match) {
+      result = url_table[i].handler(conn, &args);
+      goto done;
+    }
+  }
+
+  /* we didn't recognize the url */
+  write_short_http_response(conn, 404, "Not found");
+  result = 0;
+
+ done:
+  tor_free(url_mem);
+  return result;
+}
+
+/** Helper function for GET / or GET /tor/
+ */
+static int
+handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  (void) args; /* unused */
+  const char *frontpage = get_dirportfrontpage();
+
+  if (frontpage) {
+    size_t dlen;
+    dlen = strlen(frontpage);
+    /* Let's return a disclaimer page (users shouldn't use V1 anymore,
+       and caches don't fetch '/', so this is safe). */
+
+    /* [We don't check for write_bucket_low here, since we want to serve
+     *  this page no matter what.] */
+    write_http_response_header_impl(conn, dlen, "text/html", "identity",
+                                    NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
+    connection_buf_add(frontpage, dlen, TO_CONN(conn));
+  } else {
+    write_short_http_response(conn, 404, "Not found");
+  }
+  return 0;
+}
+
+/** Warn that the cached consensus <b>consensus</b> of type
+ * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
+ * warning to avoid logging an entry on every request.
+ */
+static void
+warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now)
+{
+#define TOO_OLD_WARNING_INTERVAL (60*60)
+  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+  char timestamp[ISO_TIME_LEN+1];
+  time_t valid_until;
+  char *dupes;
+
+  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
+    return;
+
+  if ((dupes = rate_limit_log(&warned, now))) {
+    format_local_iso_time(timestamp, valid_until);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
+             "serve it to clients. It was valid until %s local time and we "
+             "continued to serve it for up to 24 hours after it expired.%s",
+             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+    tor_free(dupes);
+  }
+}
+
+/**
+ * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
+ * <b>digest</b>. Return 0 on success.  On failure, report that the hash came
+ * from <b>location</b>, report that we are taking <b>action</b> with it, and
+ * return -1.
+ */
+static int
+parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
+                    const char *action)
+{
+  if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
+      DIGEST256_LEN) {
+    return 0;
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_DIR,
+           "%s contained bogus digest %s; %s.",
+           location, escaped(hex), action);
+    return -1;
+  }
+}
+
+/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
+ * set <b>digest_out<b> to a new smartlist containing every 256-bit
+ * hex-encoded digest listed in that header and return 0.  Otherwise return
+ * -1.  */
+static int
+parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
+{
+  char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
+  if (hdr == NULL) {
+    return -1;
+  }
+  smartlist_t *hex_digests = smartlist_new();
+  *digests_out = smartlist_new();
+  smartlist_split_string(hex_digests, hdr, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
+    uint8_t digest[DIGEST256_LEN];
+    if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
+                             "ignoring")) {
+      smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
+    }
+  } SMARTLIST_FOREACH_END(hex);
+  SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
+  smartlist_free(hex_digests);
+  tor_free(hdr);
+  return 0;
+}
+
+/** Fallback compression method.  The fallback compression method is used in
+ * case a client requests a non-compressed document. We only store compressed
+ * documents, so we use this compression method to fetch the document and let
+ * the spooling system do the streaming decompression.
+ */
+#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
+
+/**
+ * Try to find the best consensus diff possible in order to serve a client
+ * request for a diff from one of the consensuses in <b>digests</b> to the
+ * current consensus of flavor <b>flav</b>.  The client supports the
+ * compression methods listed in the <b>compression_methods</b> bitfield:
+ * place the method chosen (if any) into <b>compression_used_out</b>.
+ */
+static struct consensus_cache_entry_t *
+find_best_diff(const smartlist_t *digests, int flav,
+               unsigned compression_methods,
+               compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    unsigned u;
+    for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+      compress_method_t method = srv_meth_pref_precompressed[u];
+      if (0 == (compression_methods & (1u<<method)))
+        continue; // client doesn't like this one, or we don't have it.
+      if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
+                                     diff_from, DIGEST256_LEN,
+                                     method) == CONSDIFF_AVAILABLE) {
+        tor_assert_nonfatal(result);
+        *compression_used_out = method;
+        return result;
+      }
+    }
+  } SMARTLIST_FOREACH_END(diff_from);
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
+          DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = FALLBACK_COMPRESS_METHOD;
+      return result;
+    }
+  } SMARTLIST_FOREACH_END(diff_from);
+
+  return NULL;
+}
+
+/** Lookup the cached consensus document by the flavor found in <b>flav</b>.
+ * The preferred set of compression methods should be listed in the
+ * <b>compression_methods</b> bitfield. The compression method chosen (if any)
+ * is stored in <b>compression_used_out</b>. */
+static struct consensus_cache_entry_t *
+find_best_consensus(int flav,
+                    unsigned compression_methods,
+                    compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+  unsigned u;
+
+  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+    compress_method_t method = srv_meth_pref_precompressed[u];
+
+    if (0 == (compression_methods & (1u<<method)))
+      continue;
+
+    if (consdiffmgr_find_consensus(&result, flav,
+                                   method) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = method;
+      return result;
+    }
+  }
+
+  if (consdiffmgr_find_consensus(&result, flav,
+        FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+    tor_assert_nonfatal(result);
+    *compression_used_out = FALLBACK_COMPRESS_METHOD;
+    return result;
+  }
+
+  return NULL;
+}
+
+/** Try to find the best supported compression method possible from a given
+ * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
+ * compression method could be found. */
+static compress_method_t
+find_best_compression_method(unsigned compression_methods, int stream)
+{
+  unsigned u;
+  compress_method_t *methods;
+  size_t length;
+
+  if (stream) {
+    methods = srv_meth_pref_streaming_compression;
+    length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
+  } else {
+    methods = srv_meth_pref_precompressed;
+    length = ARRAY_LENGTH(srv_meth_pref_precompressed);
+  }
+
+  for (u = 0; u < length; ++u) {
+    compress_method_t method = methods[u];
+    if (compression_methods & (1u<<method))
+      return method;
+  }
+
+  return NO_METHOD;
+}
+
+/** Check if any of the digests in <b>digests</b> matches the latest consensus
+ *  flavor (given in <b>flavor</b>) that we have available. */
+static int
+digest_list_contains_best_consensus(consensus_flavor_t flavor,
+                                    const smartlist_t *digests)
+{
+  const networkstatus_t *ns = NULL;
+
+  if (digests == NULL)
+    return 0;
+
+  ns = networkstatus_get_latest_consensus_by_flavor(flavor);
+
+  if (ns == NULL)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
+    if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN))
+      return 1;
+  } SMARTLIST_FOREACH_END(digest);
+
+  return 0;
+}
+
+/** Check if the given compression method is allowed for a connection that is
+ * supposed to be anonymous. Returns 1 if the compression method is allowed,
+ * otherwise 0. */
+STATIC int
+allowed_anonymous_connection_compression_method(compress_method_t method)
+{
+  unsigned u;
+
+  for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression);
+       ++u) {
+    compress_method_t allowed_method =
+      client_meth_allowed_anonymous_compression[u];
+
+    if (! tor_compress_supports_method(allowed_method))
+      continue;
+
+    if (method == allowed_method)
+      return 1;
+  }
+
+  return 0;
+}
+
+/** Log a warning when a remote server has sent us a document using a
+ * compression method that is not allowed for anonymous directory requests. */
+STATIC void
+warn_disallowed_anonymous_compression_method(compress_method_t method)
+{
+  log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
+         "Received a %s HTTP response, which is not "
+         "allowed for anonymous directory requests.",
+         compression_method_get_human_name(method));
+}
+
+/** Encodes the results of parsing a consensus request to figure out what
+ * consensus, and possibly what diffs, the user asked for. */
+typedef struct {
+  /** name of the flavor to retrieve. */
+  char *flavor;
+  /** flavor to retrive, as enum. */
+  consensus_flavor_t flav;
+  /** plus-separated list of authority fingerprints; see
+   * client_likes_consensus(). Aliases the URL in the request passed to
+   * parse_consensus_request(). */
+  const char *want_fps;
+  /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
+   * to return a diff from. */
+  smartlist_t *diff_from_digests;
+  /** If true, never send a full consensus. If there is no diff, send
+   * a 404 instead. */
+  int diff_only;
+} parsed_consensus_request_t;
+
+/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
+ * it is stack-allocated. */
+static void
+parsed_consensus_request_clear(parsed_consensus_request_t *req)
+{
+  if (!req)
+    return;
+  tor_free(req->flavor);
+  if (req->diff_from_digests) {
+    SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
+    smartlist_free(req->diff_from_digests);
+  }
+  memset(req, 0, sizeof(parsed_consensus_request_t));
+}
+
+/**
+ * Parse the URL and relevant headers of <b>args</b> for a current-consensus
+ * request to learn what flavor of consensus we want, what keys it must be
+ * signed with, and what diffs we would accept (or demand) instead. Return 0
+ * on success and -1 on failure.
+ */
+static int
+parse_consensus_request(parsed_consensus_request_t *out,
+                        const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  memset(out, 0, sizeof(parsed_consensus_request_t));
+  out->flav = FLAV_NS;
+
+  const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
+  const char CONSENSUS_FLAVORED_PREFIX[] =
+    "/tor/status-vote/current/consensus-";
+
+  /* figure out the flavor if any, and who we wanted to sign the thing */
+  const char *after_flavor = NULL;
+
+  if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
+    const char *f, *cp;
+    f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
+    cp = strchr(f, '/');
+    if (cp) {
+      after_flavor = cp+1;
+      out->flavor = tor_strndup(f, cp-f);
+    } else {
+      out->flavor = tor_strdup(f);
+    }
+    int flav = networkstatus_parse_flavor_name(out->flavor);
+    if (flav < 0)
+      flav = FLAV_NS;
+    out->flav = flav;
+  } else {
+    if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
+      after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
+  }
+
+  /* see whether we've been asked explicitly for a diff from an older
+   * consensus. (The user might also have said that a diff would be okay,
+   * via X-Or-Diff-From-Consensus */
+  const char DIFF_COMPONENT[] = "diff/";
+  char *diff_hash_in_url = NULL;
+  if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
+    after_flavor += strlen(DIFF_COMPONENT);
+    const char *cp = strchr(after_flavor, '/');
+    if (cp) {
+      diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
+      out->want_fps = cp+1;
+    } else {
+      diff_hash_in_url = tor_strdup(after_flavor);
+      out->want_fps = NULL;
+    }
+  } else {
+    out->want_fps = after_flavor;
+  }
+
+  if (diff_hash_in_url) {
+    uint8_t diff_from[DIGEST256_LEN];
+    out->diff_from_digests = smartlist_new();
+    out->diff_only = 1;
+    int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
+                                  "rejecting");
+    tor_free(diff_hash_in_url);
+    if (ok) {
+      smartlist_add(out->diff_from_digests,
+                    tor_memdup(diff_from, DIGEST256_LEN));
+    } else {
+      return -1;
+    }
+  } else {
+    parse_or_diff_from_header(&out->diff_from_digests, args->headers);
+  }
+
+  return 0;
+}
+
+/** Helper function for GET /tor/status-vote/current/consensus
+ */
+static int
+handle_get_current_consensus(dir_connection_t *conn,
+                             const get_handler_args_t *args)
+{
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 0);
+  const time_t if_modified_since = args->if_modified_since;
+  int clear_spool = 0;
+
+  /* v3 network status fetch. */
+  long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
+
+  time_t now = time(NULL);
+  parsed_consensus_request_t req;
+
+  if (parse_consensus_request(&req, args) < 0) {
+    write_short_http_response(conn, 404, "Couldn't parse request");
+    goto done;
+  }
+
+  if (digest_list_contains_best_consensus(req.flav,
+                                          req.diff_from_digests)) {
+    write_short_http_response(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+    goto done;
+  }
+
+  struct consensus_cache_entry_t *cached_consensus = NULL;
+
+  compress_method_t compression_used = NO_METHOD;
+  if (req.diff_from_digests) {
+    cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
+                                      args->compression_supported,
+                                      &compression_used);
+  }
+
+  if (req.diff_only && !cached_consensus) {
+    write_short_http_response(conn, 404, "No such diff available");
+    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (! cached_consensus) {
+    cached_consensus = find_best_consensus(req.flav,
+                                           args->compression_supported,
+                                           &compression_used);
+  }
+
+  time_t fresh_until, valid_until;
+  int have_fresh_until = 0, have_valid_until = 0;
+  if (cached_consensus) {
+    have_fresh_until =
+      !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
+    have_valid_until =
+      !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
+  }
+
+  if (cached_consensus && have_valid_until &&
+      !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
+    write_short_http_response(conn, 404, "Consensus is too old");
+    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (cached_consensus && req.want_fps &&
+      !client_likes_consensus(cached_consensus, req.want_fps)) {
+    write_short_http_response(conn, 404, "Consensus not signed by sufficient "
+                           "number of requested authorities");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
+    goto done;
+  }
+
+  conn->spool = smartlist_new();
+  clear_spool = 1;
+  {
+    spooled_resource_t *spooled;
+    if (cached_consensus) {
+      spooled = spooled_resource_new_from_cache_entry(cached_consensus);
+      smartlist_add(conn->spool, spooled);
+    }
+  }
+
+  lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
+
+  size_t size_guess = 0;
+  int n_expired = 0;
+  dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
+                                              compress_method != NO_METHOD,
+                                              &size_guess,
+                                              &n_expired);
+
+  if (!smartlist_len(conn->spool) && !n_expired) {
+    write_short_http_response(conn, 404, "Not found");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (!smartlist_len(conn->spool)) {
+    write_short_http_response(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+    goto done;
+  }
+
+  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    log_debug(LD_DIRSERV,
+              "Client asked for network status lists, but we've been "
+              "writing too many bytes lately. Sending 503 Dir busy.");
+    write_short_http_response(conn, 503, "Directory busy, try again later");
+    geoip_note_ns_response(GEOIP_REJECT_BUSY);
+    goto done;
+  }
+
+  tor_addr_t addr;
+  if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
+    geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
+                           &addr, NULL,
+                           time(NULL));
+    geoip_note_ns_response(GEOIP_SUCCESS);
+    /* Note that a request for a network status has started, so that we
+     * can measure the download time later on. */
+    if (conn->dirreq_id)
+      geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
+    else
+      geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
+                         DIRREQ_DIRECT);
+  }
+
+  /* Use this header to tell caches that the response depends on the
+   * X-Or-Diff-From-Consensus header (or lack thereof). */
+  const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
+
+  clear_spool = 0;
+
+  // The compress_method might have been NO_METHOD, but we store the data
+  // compressed. Decompress them using `compression_used`. See fallback code in
+  // find_best_consensus() and find_best_diff().
+  write_http_response_headers(conn, -1,
+                             compress_method == NO_METHOD ?
+                               NO_METHOD : compression_used,
+                             vary_header,
+                             smartlist_len(conn->spool) == 1 ? lifetime : 0);
+
+  if (compress_method == NO_METHOD && smartlist_len(conn->spool))
+    conn->compress_state = tor_compress_new(0, compression_used,
+                                            HIGH_COMPRESSION);
+
+  /* Prime the connection with some data. */
+  const int initial_flush_result = connection_dirserv_flushed_some(conn);
+  tor_assert_nonfatal(initial_flush_result == 0);
+  goto done;
+
+ done:
+  parsed_consensus_request_clear(&req);
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
+  return 0;
+}
+
+/** Helper function for GET /tor/status-vote/{current,next}/...
+ */
+static int
+handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  {
+    ssize_t body_len = 0;
+    ssize_t estimated_len = 0;
+    int lifetime = 60; /* XXXX?? should actually use vote intervals. */
+    /* This smartlist holds strings that we can compress on the fly. */
+    smartlist_t *items = smartlist_new();
+    /* This smartlist holds cached_dir_t objects that have a precompressed
+     * deflated version. */
+    smartlist_t *dir_items = smartlist_new();
+    dirvote_dirreq_get_status_vote(url, items, dir_items);
+    if (!smartlist_len(dir_items) && !smartlist_len(items)) {
+      write_short_http_response(conn, 404, "Not found");
+      goto vote_done;
+    }
+
+    /* We're sending items from at most one kind of source */
+    tor_assert_nonfatal(smartlist_len(items) == 0 ||
+                        smartlist_len(dir_items) == 0);
+
+    int streaming;
+    unsigned mask;
+    if (smartlist_len(items)) {
+      /* We're taking strings and compressing them on the fly. */
+      streaming = 1;
+      mask = ~0u;
+    } else {
+      /* We're taking cached_dir_t objects. We only have them uncompressed
+       * or deflated. */
+      streaming = 0;
+      mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
+    }
+    const compress_method_t compress_method = find_best_compression_method(
+                              args->compression_supported&mask, streaming);
+
+    SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
+                      body_len += compress_method != NO_METHOD ?
+                        d->dir_compressed_len : d->dir_len);
+    estimated_len += body_len;
+    SMARTLIST_FOREACH(items, const char *, item, {
+        size_t ln = strlen(item);
+        if (compress_method != NO_METHOD) {
+          estimated_len += ln/2;
+        } else {
+          body_len += ln; estimated_len += ln;
+        }
+      });
+
+    if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
+      write_short_http_response(conn, 503, "Directory busy, try again later");
+      goto vote_done;
+    }
+    write_http_response_header(conn, body_len ? body_len : -1,
+                 compress_method,
+                 lifetime);
+
+    if (smartlist_len(items)) {
+      if (compress_method != NO_METHOD) {
+        conn->compress_state = tor_compress_new(1, compress_method,
+                           choose_compression_level(estimated_len));
+        SMARTLIST_FOREACH(items, const char *, c,
+                 connection_buf_add_compress(c, strlen(c), conn, 0));
+        connection_buf_add_compress("", 0, conn, 1);
+      } else {
+        SMARTLIST_FOREACH(items, const char *, c,
+                         connection_buf_add(c, strlen(c), TO_CONN(conn)));
+      }
+    } else {
+      SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
+          connection_buf_add(compress_method != NO_METHOD ?
+                                    d->dir_compressed : d->dir,
+                                  compress_method != NO_METHOD ?
+                                    d->dir_compressed_len : d->dir_len,
+                                  TO_CONN(conn)));
+    }
+  vote_done:
+    smartlist_free(items);
+    smartlist_free(dir_items);
+    goto done;
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/micro/d/...
+ */
+static int
+handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  int clear_spool = 1;
+  {
+    conn->spool = smartlist_new();
+
+    dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
+                                      DIR_SPOOL_MICRODESC,
+                                      conn->spool, NULL,
+                                      DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
+
+    size_t size_guess = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, 0,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, NULL);
+    if (smartlist_len(conn->spool) == 0) {
+      write_short_http_response(conn, 404, "Not found");
+      goto done;
+    }
+    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+      log_info(LD_DIRSERV,
+               "Client asked for server descriptors, but we've been "
+               "writing too many bytes lately. Sending 503 Dir busy.");
+      write_short_http_response(conn, 503, "Directory busy, try again later");
+      goto done;
+    }
+
+    clear_spool = 0;
+    write_http_response_header(conn, -1,
+                               compress_method,
+                               MICRODESC_CACHE_LIFETIME);
+
+    if (compress_method != NO_METHOD)
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                      choose_compression_level(size_guess));
+
+    const int initial_flush_result = connection_dirserv_flushed_some(conn);
+    tor_assert_nonfatal(initial_flush_result == 0);
+    goto done;
+  }
+
+ done:
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
+  return 0;
+}
+
+/** Helper function for GET /tor/{server,extra}/...
+ */
+static int
+handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  const or_options_t *options = get_options();
+  int clear_spool = 1;
+  if (!strcmpstart(url,"/tor/server/") ||
+      (!options->BridgeAuthoritativeDir &&
+       !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
+    int res;
+    const char *msg = NULL;
+    int cache_lifetime = 0;
+    int is_extra = !strcmpstart(url,"/tor/extra/");
+    url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
+    dir_spool_source_t source;
+    time_t publish_cutoff = 0;
+    if (!strcmpstart(url, "d/")) {
+      source =
+        is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
+    } else {
+      source =
+        is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
+      /* We only want to apply a publish cutoff when we're requesting
+       * resources by fingerprint. */
+      publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+    }
+
+    conn->spool = smartlist_new();
+    res = dirserv_get_routerdesc_spool(conn->spool, url,
+                                       source,
+                                       connection_dir_is_encrypted(conn),
+                                       &msg);
+
+    if (!strcmpstart(url, "all")) {
+      cache_lifetime = FULL_DIR_CACHE_LIFETIME;
+    } else if (smartlist_len(conn->spool) == 1) {
+      cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
+    }
+
+    size_t size_guess = 0;
+    int n_expired = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, &n_expired);
+
+    /* If we are the bridge authority and the descriptor is a bridge
+     * descriptor, remember that we served this descriptor for desc stats. */
+    /* XXXX it's a bit of a kludge to have this here. */
+    if (get_options()->BridgeAuthoritativeDir &&
+        source == DIR_SPOOL_SERVER_BY_FP) {
+      SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
+        const routerinfo_t *router =
+          router_get_by_id_digest((const char *)spooled->digest);
+        /* router can be NULL here when the bridge auth is asked for its own
+         * descriptor. */
+        if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
+          rep_hist_note_desc_served(router->cache_info.identity_digest);
+      } SMARTLIST_FOREACH_END(spooled);
+    }
+
+    if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
+      if (msg == NULL)
+        msg = "Not found";
+      write_short_http_response(conn, 404, msg);
+    } else {
+      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+        log_info(LD_DIRSERV,
+                 "Client asked for server descriptors, but we've been "
+                 "writing too many bytes lately. Sending 503 Dir busy.");
+        write_short_http_response(conn, 503,
+                                  "Directory busy, try again later");
+        dir_conn_clear_spool(conn);
+        goto done;
+      }
+      write_http_response_header(conn, -1, compress_method, cache_lifetime);
+      if (compress_method != NO_METHOD)
+        conn->compress_state = tor_compress_new(1, compress_method,
+                                        choose_compression_level(size_guess));
+      clear_spool = 0;
+      /* Prime the connection with some data. */
+      int initial_flush_result = connection_dirserv_flushed_some(conn);
+      tor_assert_nonfatal(initial_flush_result == 0);
+    }
+    goto done;
+  }
+ done:
+  if (clear_spool)
+    dir_conn_clear_spool(conn);
+  return 0;
+}
+
+/** Helper function for GET /tor/keys/...
+ */
+static int
+handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  const time_t if_modified_since = args->if_modified_since;
+  {
+    smartlist_t *certs = smartlist_new();
+    ssize_t len = -1;
+    if (!strcmp(url, "/tor/keys/all")) {
+      authority_cert_get_all(certs);
+    } else if (!strcmp(url, "/tor/keys/authority")) {
+      authority_cert_t *cert = get_my_v3_authority_cert();
+      if (cert)
+        smartlist_add(certs, cert);
+    } else if (!strcmpstart(url, "/tor/keys/fp/")) {
+      smartlist_t *fps = smartlist_new();
+      dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
+                                           fps, NULL,
+                                           DSR_HEX|DSR_SORT_UNIQ);
+      SMARTLIST_FOREACH(fps, char *, d, {
+          authority_cert_t *c = authority_cert_get_newest_by_id(d);
+          if (c) smartlist_add(certs, c);
+          tor_free(d);
+      });
+      smartlist_free(fps);
+    } else if (!strcmpstart(url, "/tor/keys/sk/")) {
+      smartlist_t *fps = smartlist_new();
+      dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
+                                           fps, NULL,
+                                           DSR_HEX|DSR_SORT_UNIQ);
+      SMARTLIST_FOREACH(fps, char *, d, {
+          authority_cert_t *c = authority_cert_get_by_sk_digest(d);
+          if (c) smartlist_add(certs, c);
+          tor_free(d);
+      });
+      smartlist_free(fps);
+    } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
+      smartlist_t *fp_sks = smartlist_new();
+      dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
+                                                fp_sks);
+      SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
+          authority_cert_t *c = authority_cert_get_by_digests(pair->first,
+                                                              pair->second);
+          if (c) smartlist_add(certs, c);
+          tor_free(pair);
+      });
+      smartlist_free(fp_sks);
+    } else {
+      write_short_http_response(conn, 400, "Bad request");
+      goto keys_done;
+    }
+    if (!smartlist_len(certs)) {
+      write_short_http_response(conn, 404, "Not found");
+      goto keys_done;
+    }
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+      if (c->cache_info.published_on < if_modified_since)
+        SMARTLIST_DEL_CURRENT(certs, c));
+    if (!smartlist_len(certs)) {
+      write_short_http_response(conn, 304, "Not modified");
+      goto keys_done;
+    }
+    len = 0;
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+                      len += c->cache_info.signed_descriptor_len);
+
+    if (global_write_bucket_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len,
+                                2)) {
+      write_short_http_response(conn, 503, "Directory busy, try again later");
+      goto keys_done;
+    }
+
+    write_http_response_header(conn,
+                               compress_method != NO_METHOD ? -1 : len,
+                               compress_method,
+                               60*60);
+    if (compress_method != NO_METHOD) {
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                              choose_compression_level(len));
+      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+            connection_buf_add_compress(
+                c->cache_info.signed_descriptor_body,
+                c->cache_info.signed_descriptor_len,
+                conn, 0));
+      connection_buf_add_compress("", 0, conn, 1);
+    } else {
+      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+            connection_buf_add(c->cache_info.signed_descriptor_body,
+                                    c->cache_info.signed_descriptor_len,
+                                    TO_CONN(conn)));
+    }
+  keys_done:
+    smartlist_free(certs);
+    goto done;
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/rendezvous2/
+ */
+static int
+handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                            const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  if (connection_dir_is_encrypted(conn)) {
+    /* Handle v2 rendezvous descriptor fetch request. */
+    const char *descp;
+    const char *query = url + strlen("/tor/rendezvous2/");
+    if (rend_valid_descriptor_id(query)) {
+      log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
+               safe_str(escaped(query)));
+      switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
+        case 1: /* valid */
+          write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
+          connection_buf_add(descp, strlen(descp), TO_CONN(conn));
+          break;
+        case 0: /* well-formed but not present */
+          write_short_http_response(conn, 404, "Not found");
+          break;
+        case -1: /* not well-formed */
+          write_short_http_response(conn, 400, "Bad request");
+          break;
+      }
+    } else { /* not well-formed */
+      write_short_http_response(conn, 400, "Bad request");
+    }
+    goto done;
+  } else {
+    /* Not encrypted! */
+    write_short_http_response(conn, 404, "Not found");
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/hs/3/<z>. Only for version 3.
+ */
+STATIC int
+handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                            const get_handler_args_t *args)
+{
+  int retval;
+  const char *desc_str = NULL;
+  const char *pubkey_str = NULL;
+  const char *url = args->url;
+
+  /* Reject unencrypted dir connections */
+  if (!connection_dir_is_encrypted(conn)) {
+    write_short_http_response(conn, 404, "Not found");
+    goto done;
+  }
+
+  /* After the path prefix follows the base64 encoded blinded pubkey which we
+   * use to get the descriptor from the cache. Skip the prefix and get the
+   * pubkey. */
+  tor_assert(!strcmpstart(url, "/tor/hs/3/"));
+  pubkey_str = url + strlen("/tor/hs/3/");
+  retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
+                                  pubkey_str, &desc_str);
+  if (retval <= 0 || desc_str == NULL) {
+    write_short_http_response(conn, 404, "Not found");
+    goto done;
+  }
+
+  /* Found requested descriptor! Pass it to this nice client. */
+  write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
+  connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
+
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/networkstatus-bridges
+ */
+static int
+handle_get_networkstatus_bridges(dir_connection_t *conn,
+                                 const get_handler_args_t *args)
+{
+  const char *headers = args->headers;
+
+  const or_options_t *options = get_options();
+  if (options->BridgeAuthoritativeDir &&
+      options->BridgePassword_AuthDigest_ &&
+      connection_dir_is_encrypted(conn)) {
+    char *status;
+    char digest[DIGEST256_LEN];
+
+    char *header = http_get_header(headers, "Authorization: Basic ");
+    if (header)
+      crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
+
+    /* now make sure the password is there and right */
+    if (!header ||
+        tor_memneq(digest,
+                   options->BridgePassword_AuthDigest_, DIGEST256_LEN)) {
+      write_short_http_response(conn, 404, "Not found");
+      tor_free(header);
+      goto done;
+    }
+    tor_free(header);
+
+    /* all happy now. send an answer. */
+    status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
+    size_t dlen = strlen(status);
+    write_http_response_header(conn, dlen, NO_METHOD, 0);
+    connection_buf_add(status, dlen, TO_CONN(conn));
+    tor_free(status);
+    goto done;
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET robots.txt or /tor/robots.txt */
+static int
+handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  (void)args;
+  {
+    const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
+    size_t len = strlen(robots);
+    write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
+    connection_buf_add(robots, len, TO_CONN(conn));
+  }
+  return 0;
+}
+
+/* Given the <b>url</b> from a POST request, try to extract the version number
+ * using the provided <b>prefix</b>. The version should be after the prefix and
+ * ending with the separator "/". For instance:
+ *      /tor/hs/3/publish
+ *
+ * On success, <b>end_pos</b> points to the position right after the version
+ * was found. On error, it is set to NULL.
+ *
+ * Return version on success else negative value. */
+STATIC int
+parse_hs_version_from_post(const char *url, const char *prefix,
+                           const char **end_pos)
+{
+  int ok;
+  unsigned long version;
+  const char *start;
+  char *end = NULL;
+
+  tor_assert(url);
+  tor_assert(prefix);
+  tor_assert(end_pos);
+
+  /* Check if the prefix does start the url. */
+  if (strcmpstart(url, prefix)) {
+    goto err;
+  }
+  /* Move pointer to the end of the prefix string. */
+  start = url + strlen(prefix);
+  /* Try this to be the HS version and if we are still at the separator, next
+   * will be move to the right value. */
+  version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
+  if (!ok) {
+    goto err;
+  }
+
+  *end_pos = end;
+  return (int) version;
+ err:
+  *end_pos = NULL;
+  return -1;
+}
+
+/* Handle the POST request for a hidden service descripror. The request is in
+ * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
+ * else return 400 indicating a bad request. */
+STATIC int
+handle_post_hs_descriptor(const char *url, const char *body)
+{
+  int version;
+  const char *end_pos;
+
+  tor_assert(url);
+  tor_assert(body);
+
+  version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
+  if (version < 0) {
+    goto err;
+  }
+
+  /* We have a valid version number, now make sure it's a publish request. Use
+   * the end position just after the version and check for the command. */
+  if (strcmpstart(end_pos, "/publish")) {
+    goto err;
+  }
+
+  switch (version) {
+  case HS_VERSION_THREE:
+    if (hs_cache_store_as_dir(body) < 0) {
+      goto err;
+    }
+    log_info(LD_REND, "Publish request for HS descriptor handled "
+                      "successfully.");
+    break;
+  default:
+    /* Unsupported version, return a bad request. */
+    goto err;
+  }
+
+  return 200;
+ err:
+  /* Bad request. */
+  return 400;
+}
+
+/** Helper function: called when a dirserver gets a complete HTTP POST
+ * request.  Look for an uploaded server descriptor or rendezvous
+ * service descriptor.  On finding one, process it and write a
+ * response into conn-\>outbuf.  If the request is unrecognized, send a
+ * 400.  Always return 0. */
+MOCK_IMPL(STATIC int,
+directory_handle_command_post,(dir_connection_t *conn, const char *headers,
+                               const char *body, size_t body_len))
+{
+  char *url = NULL;
+  const or_options_t *options = get_options();
+
+  log_debug(LD_DIRSERV,"Received POST command.");
+
+  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
+
+  if (!public_server_mode(options)) {
+    log_info(LD_DIR, "Rejected dir post request from %s "
+             "since we're not a public relay.", conn->base_.address);
+    write_short_http_response(conn, 503, "Not acting as a public relay");
+    goto done;
+  }
+
+  if (parse_http_url(headers, &url) < 0) {
+    write_short_http_response(conn, 400, "Bad request");
+    return 0;
+  }
+  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
+
+  /* Handle v2 rendezvous service publish request. */
+  if (connection_dir_is_encrypted(conn) &&
+      !strcmpstart(url,"/tor/rendezvous2/publish")) {
+    if (rend_cache_store_v2_desc_as_dir(body) < 0) {
+      log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
+               (int)body_len, conn->base_.address);
+      write_short_http_response(conn, 400,
+                             "Invalid v2 service descriptor rejected");
+    } else {
+      write_short_http_response(conn, 200, "Service descriptor (v2) stored");
+      log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
+    }
+    goto done;
+  }
+
+  /* Handle HS descriptor publish request. */
+  /* XXX: This should be disabled with a consensus param until we want to
+   * the prop224 be deployed and thus use. */
+  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+    const char *msg = "HS descriptor stored successfully.";
+
+    /* We most probably have a publish request for an HS descriptor. */
+    int code = handle_post_hs_descriptor(url, body);
+    if (code != 200) {
+      msg = "Invalid HS descriptor. Rejected.";
+    }
+    write_short_http_response(conn, code, msg);
+    goto done;
+  }
+
+  if (!authdir_mode(options)) {
+    /* we just provide cached directories; we don't want to
+     * receive anything. */
+    write_short_http_response(conn, 400, "Nonauthoritative directory does not "
+                           "accept posted server descriptors");
+    goto done;
+  }
+
+  if (authdir_mode(options) &&
+      !strcmp(url,"/tor/")) { /* server descriptor post */
+    const char *msg = "[None]";
+    uint8_t purpose = authdir_mode_bridge(options) ?
+                      ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
+    was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
+                                             conn->base_.address, &msg);
+    tor_assert(msg);
+
+    if (r == ROUTER_ADDED_SUCCESSFULLY) {
+      write_short_http_response(conn, 200, msg);
+    } else if (WRA_WAS_OUTDATED(r)) {
+      write_http_response_header_impl(conn, -1, NULL, NULL,
+                                      "X-Descriptor-Not-New: Yes\r\n", -1);
+    } else {
+      log_info(LD_DIRSERV,
+               "Rejected router descriptor or extra-info from %s "
+               "(\"%s\").",
+               conn->base_.address, msg);
+      write_short_http_response(conn, 400, msg);
+    }
+    goto done;
+  }
+
+  if (authdir_mode_v3(options) &&
+      !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
+    const char *msg = "OK";
+    int status;
+    if (dirvote_add_vote(body, &msg, &status)) {
+      write_short_http_response(conn, status, "Vote stored");
+    } else {
+      tor_assert(msg);
+      log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
+               conn->base_.address, msg);
+      write_short_http_response(conn, status, msg);
+    }
+    goto done;
+  }
+
+  if (authdir_mode_v3(options) &&
+      !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
+    const char *msg = NULL;
+    if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
+      write_short_http_response(conn, 200, msg?msg:"Signatures stored");
+    } else {
+      log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
+               conn->base_.address, msg?msg:"???");
+      write_short_http_response(conn, 400,
+                                msg?msg:"Unable to store signatures");
+    }
+    goto done;
+  }
+
+  /* we didn't recognize the url */
+  write_short_http_response(conn, 404, "Not found");
+
+ done:
+  tor_free(url);
+  return 0;
+}
+
+/** Called when a dirserver receives data on a directory connection;
+ * looks for an HTTP request.  If the request is complete, remove it
+ * from the inbuf, try to process it; otherwise, leave it on the
+ * buffer.  Return a 0 on success, or -1 on error.
+ */
+STATIC int
+directory_handle_command(dir_connection_t *conn)
+{
+  char *headers=NULL, *body=NULL;
+  size_t body_len=0;
+  int r;
+
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  switch (connection_fetch_from_buf_http(TO_CONN(conn),
+                              &headers, MAX_HEADERS_SIZE,
+                              &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
+    case -1: /* overflow */
+      log_warn(LD_DIRSERV,
+               "Request too large from address '%s' to DirPort. Closing.",
+               safe_str(conn->base_.address));
+      return -1;
+    case 0:
+      log_debug(LD_DIRSERV,"command not all here yet.");
+      return 0;
+    /* case 1, fall through */
+  }
+
+  http_set_address_origin(headers, TO_CONN(conn));
+  // we should escape headers here as well,
+  // but we can't call escaped() twice, as it uses the same buffer
+  //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
+
+  if (!strncasecmp(headers,"GET",3))
+    r = directory_handle_command_get(conn, headers, body, body_len);
+  else if (!strncasecmp(headers,"POST",4))
+    r = directory_handle_command_post(conn, headers, body, body_len);
+  else {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Got headers %s with unknown command. Closing.",
+           escaped(headers));
+    r = -1;
+  }
+
+  tor_free(headers); tor_free(body);
+  return r;
+}
+
+/** Write handler for directory connections; called when all data has
+ * been flushed.  Close the connection or wait for a response as
+ * appropriate.
+ */
+int
+connection_dir_finished_flushing(dir_connection_t *conn)
+{
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  /* Note that we have finished writing the directory response. For direct
+   * connections this means we're done; for tunneled connections it's only
+   * an intermediate step. */
+  if (conn->dirreq_id)
+    geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED,
+                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
+  else
+    geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,
+                              DIRREQ_DIRECT,
+                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
+  switch (conn->base_.state) {
+    case DIR_CONN_STATE_CONNECTING:
+    case DIR_CONN_STATE_CLIENT_SENDING:
+      log_debug(LD_DIR,"client finished sending command.");
+      conn->base_.state = DIR_CONN_STATE_CLIENT_READING;
+      return 0;
+    case DIR_CONN_STATE_SERVER_WRITING:
+      if (conn->spool) {
+        log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
+        connection_mark_for_close(TO_CONN(conn));
+      } else {
+        log_debug(LD_DIRSERV, "Finished writing server response. Closing.");
+        connection_mark_for_close(TO_CONN(conn));
+      }
+      return 0;
+    default:
+      log_warn(LD_BUG,"called in unexpected state %d.",
+               conn->base_.state);
+      tor_fragile_assert();
+      return -1;
+  }
+  return 0;
+}
+
+/* We just got a new consensus! If there are other in-progress requests
+ * for this consensus flavor (for example because we launched several in
+ * parallel), cancel them.
+ *
+ * We do this check here (not just in
+ * connection_ap_handshake_attach_circuit()) to handle the edge case where
+ * a consensus fetch begins and ends before some other one tries to attach to
+ * a circuit, in which case the other one won't know that we're all happy now.
+ *
+ * Don't mark the conn that just gave us the consensus -- otherwise we
+ * would end up double-marking it when it cleans itself up.
+ */
+static void
+connection_dir_close_consensus_fetches(dir_connection_t *except_this_one,
+                                       const char *resource)
+{
+  smartlist_t *conns_to_close =
+    connection_dir_list_by_purpose_and_resource(DIR_PURPOSE_FETCH_CONSENSUS,
+                                                resource);
+  SMARTLIST_FOREACH_BEGIN(conns_to_close, dir_connection_t *, d) {
+    if (d == except_this_one)
+      continue;
+    log_info(LD_DIR, "Closing consensus fetch (to %s) since one "
+             "has just arrived.", TO_CONN(d)->address);
+    connection_mark_for_close(TO_CONN(d));
+  } SMARTLIST_FOREACH_END(d);
+  smartlist_free(conns_to_close);
+}
+
+/** Connected handler for directory connections: begin sending data to the
+ * server, and return 0.
+ * Only used when connections don't immediately connect. */
+int
+connection_dir_finished_connecting(dir_connection_t *conn)
+{
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+  tor_assert(conn->base_.state == DIR_CONN_STATE_CONNECTING);
+
+  log_debug(LD_HTTP,"Dir connection to router %s:%u established.",
+            conn->base_.address,conn->base_.port);
+
+  /* start flushing conn */
+  conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
+  return 0;
+}
+
+/** Decide which download schedule we want to use based on descriptor type
+ * in <b>dls</b> and <b>options</b>.
+ *
+ * Then, return the initial delay for that download schedule, in seconds.
+ *
+ * Helper function for download_status_increment_failure(),
+ * download_status_reset(), and download_status_increment_attempt(). */
+STATIC int
+find_dl_min_delay(const download_status_t *dls, const or_options_t *options)
+{
+  tor_assert(dls);
+  tor_assert(options);
+
+  switch (dls->schedule) {
+    case DL_SCHED_GENERIC:
+      /* Any other directory document */
+      if (dir_server_mode(options)) {
+        /* A directory authority or directory mirror */
+        return options->TestingServerDownloadInitialDelay;
+      } else {
+        return options->TestingClientDownloadInitialDelay;
+      }
+    case DL_SCHED_CONSENSUS:
+      if (!networkstatus_consensus_can_use_multiple_directories(options)) {
+        /* A public relay */
+        return options->TestingServerConsensusDownloadInitialDelay;
+      } else {
+        /* A client or bridge */
+        if (networkstatus_consensus_is_bootstrapping(time(NULL))) {
+          /* During bootstrapping */
+          if (!networkstatus_consensus_can_use_extra_fallbacks(options)) {
+            /* A bootstrapping client without extra fallback directories */
+            return options->
+              ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay;
+          } else if (dls->want_authority) {
+            /* A bootstrapping client with extra fallback directories, but
+             * connecting to an authority */
+            return
+             options->ClientBootstrapConsensusAuthorityDownloadInitialDelay;
+          } else {
+            /* A bootstrapping client connecting to extra fallback directories
+             */
+            return
+              options->ClientBootstrapConsensusFallbackDownloadInitialDelay;
+          }
+        } else {
+          /* A client with a reasonably live consensus, with or without
+           * certificates */
+          return options->TestingClientConsensusDownloadInitialDelay;
+        }
+      }
+    case DL_SCHED_BRIDGE:
+      if (options->UseBridges && num_bridges_usable(0) > 0) {
+        /* A bridge client that is sure that one or more of its bridges are
+         * running can afford to wait longer to update bridge descriptors. */
+        return options->TestingBridgeDownloadInitialDelay;
+      } else {
+        /* A bridge client which might have no running bridges, must try to
+         * get bridge descriptors straight away. */
+        return options->TestingBridgeBootstrapDownloadInitialDelay;
+      }
+    default:
+      tor_assert(0);
+  }
+
+  /* Impossible, but gcc will fail with -Werror without a `return`. */
+  return 0;
+}
+
+/** As next_random_exponential_delay() below, but does not compute a random
+ * value. Instead, compute the range of values that
+ * next_random_exponential_delay() should use when computing its random value.
+ * Store the low bound into *<b>low_bound_out</b>, and the high bound into
+ * *<b>high_bound_out</b>.  Guarantees that the low bound is strictly less
+ * than the high bound. */
+STATIC void
+next_random_exponential_delay_range(int *low_bound_out,
+                                    int *high_bound_out,
+                                    int delay,
+                                    int base_delay)
+{
+  // This is the "decorrelated jitter" approach, from
+  //    https://www.awsarchitectureblog.com/2015/03/backoff.html
+  // The formula is
+  //    sleep = min(cap, random_between(base, sleep * 3))
+
+  const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX;
+  *low_bound_out = base_delay;
+  if (delay_times_3 > base_delay) {
+    *high_bound_out = delay_times_3;
+  } else {
+    *high_bound_out = base_delay+1;
+  }
+}
+
+/** Advance one delay step.  The algorithm will generate a random delay,
+ * such that each failure is possibly (random) longer than the ones before.
+ *
+ * We then clamp that value to be no larger than max_delay, and return it.
+ *
+ * The <b>base_delay</b> parameter is lowest possible delay time (can't be
+ * zero); the <b>backoff_position</b> parameter is the number of times we've
+ * generated a delay; and the <b>delay</b> argument is the most recently used
+ * delay.
+ */
+STATIC int
+next_random_exponential_delay(int delay,
+                              int base_delay)
+{
+  /* Check preconditions */
+  if (BUG(delay < 0))
+    delay = 0;
+
+  if (base_delay < 1)
+    base_delay = 1;
+
+  int low_bound=0, high_bound=INT_MAX;
+
+  next_random_exponential_delay_range(&low_bound, &high_bound,
+                                      delay, base_delay);
+
+  return crypto_rand_int_range(low_bound, high_bound);
+}
+
+/** Find the current delay for dls based on min_delay.
+ *
+ * This function sets dls->next_attempt_at based on now, and returns the delay.
+ * Helper for download_status_increment_failure and
+ * download_status_increment_attempt. */
+STATIC int
+download_status_schedule_get_delay(download_status_t *dls,
+                                   int min_delay,
+                                   time_t now)
+{
+  tor_assert(dls);
+  /* If we're using random exponential backoff, we do need min/max delay */
+  tor_assert(min_delay >= 0);
+
+  int delay = INT_MAX;
+  uint8_t dls_schedule_position = (dls->increment_on
+                                   == DL_SCHED_INCREMENT_ATTEMPT
+                                   ? dls->n_download_attempts
+                                   : dls->n_download_failures);
+
+  /* Check if we missed a reset somehow */
+  IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) {
+    dls->last_backoff_position = 0;
+    dls->last_delay_used = 0;
+  }
+
+  if (dls_schedule_position > 0) {
+    delay = dls->last_delay_used;
+
+    while (dls->last_backoff_position < dls_schedule_position) {
+      /* Do one increment step */
+      delay = next_random_exponential_delay(delay, min_delay);
+      /* Update our position */
+      ++(dls->last_backoff_position);
+    }
+  } else {
+    /* If we're just starting out, use the minimum delay */
+    delay = min_delay;
+  }
+
+  /* Clamp it within min/max if we have them */
+  if (min_delay >= 0 && delay < min_delay) delay = min_delay;
+
+  /* Store it for next time */
+  dls->last_backoff_position = dls_schedule_position;
+  dls->last_delay_used = delay;
+
+  /* A negative delay makes no sense. Knowing that delay is
+   * non-negative allows us to safely do the wrapping check below. */
+  tor_assert(delay >= 0);
+
+  /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction
+   * that won't overflow (since delay is non-negative). */
+  if (delay < INT_MAX && now <= TIME_MAX - delay) {
+    dls->next_attempt_at = now+delay;
+  } else {
+    dls->next_attempt_at = TIME_MAX;
+  }
+
+  return delay;
+}
+
+/* Log a debug message about item, which increments on increment_action, has
+ * incremented dls_n_download_increments times. The message varies based on
+ * was_schedule_incremented (if not, not_incremented_response is logged), and
+ * the values of increment, dls_next_attempt_at, and now.
+ * Helper for download_status_increment_failure and
+ * download_status_increment_attempt. */
+static void
+download_status_log_helper(const char *item, int was_schedule_incremented,
+                           const char *increment_action,
+                           const char *not_incremented_response,
+                           uint8_t dls_n_download_increments, int increment,
+                           time_t dls_next_attempt_at, time_t now)
+{
+  if (item) {
+    if (!was_schedule_incremented)
+      log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.",
+                item, increment_action, (int)dls_n_download_increments,
+                not_incremented_response);
+    else if (increment == 0)
+      log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.",
+                item, increment_action, (int)dls_n_download_increments);
+    else if (dls_next_attempt_at < TIME_MAX)
+      log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.",
+                item, increment_action, (int)dls_n_download_increments,
+                (int)(dls_next_attempt_at-now));
+    else
+      log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.",
+                item, increment_action, (int)dls_n_download_increments);
+  }
+}
+
+/** Determine when a failed download attempt should be retried.
+ * Called when an attempt to download <b>dls</b> has failed with HTTP status
+ * <b>status_code</b>.  Increment the failure count (if the code indicates a
+ * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to
+ * an appropriate time in the future and return it.
+ * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the
+ * failure count, and return a time in the far future for the next attempt (to
+ * avoid an immediate retry). */
+time_t
+download_status_increment_failure(download_status_t *dls, int status_code,
+                                  const char *item, int server, time_t now)
+{
+  (void) status_code; // XXXX no longer used.
+  (void) server; // XXXX no longer used.
+  int increment = -1;
+  int min_delay = 0;
+
+  tor_assert(dls);
+
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
+  /* count the failure */
+  if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) {
+    ++dls->n_download_failures;
+  }
+
+  if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
+    /* We don't find out that a failure-based schedule has attempted a
+     * connection until that connection fails.
+     * We'll never find out about successful connections, but this doesn't
+     * matter, because schedules are reset after a successful download.
+     */
+    if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
+      ++dls->n_download_attempts;
+
+    /* only return a failure retry time if this schedule increments on failures
+     */
+    min_delay = find_dl_min_delay(dls, get_options());
+    increment = download_status_schedule_get_delay(dls, min_delay, now);
+  }
+
+  download_status_log_helper(item, !dls->increment_on, "failed",
+                             "concurrently", dls->n_download_failures,
+                             increment,
+                             download_status_get_next_attempt_at(dls),
+                             now);
+
+  if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) {
+    /* stop this schedule retrying on failure, it will launch concurrent
+     * connections instead */
+    return TIME_MAX;
+  } else {
+    return download_status_get_next_attempt_at(dls);
+  }
+}
+
+/** Determine when the next download attempt should be made when using an
+ * attempt-based (potentially concurrent) download schedule.
+ * Called when an attempt to download <b>dls</b> is being initiated.
+ * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an
+ * appropriate time in the future and return it.
+ * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment
+ * the attempts, and return a time in the far future (to avoid launching a
+ * concurrent attempt). */
+time_t
+download_status_increment_attempt(download_status_t *dls, const char *item,
+                                  time_t now)
+{
+  int delay = -1;
+  int min_delay = 0;
+
+  tor_assert(dls);
+
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
+  if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
+    /* this schedule should retry on failure, and not launch any concurrent
+     attempts */
+    log_warn(LD_BUG, "Tried to launch an attempt-based connection on a "
+             "failure-based schedule.");
+    return TIME_MAX;
+  }
+
+  if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
+    ++dls->n_download_attempts;
+
+  min_delay = find_dl_min_delay(dls, get_options());
+  delay = download_status_schedule_get_delay(dls, min_delay, now);
+
+  download_status_log_helper(item, dls->increment_on, "attempted",
+                             "on failure", dls->n_download_attempts,
+                             delay, download_status_get_next_attempt_at(dls),
+                             now);
+
+  return download_status_get_next_attempt_at(dls);
+}
+
+static time_t
+download_status_get_initial_delay_from_now(const download_status_t *dls)
+{
+  /* We use constant initial delays, even in exponential backoff
+   * schedules. */
+  return time(NULL) + find_dl_min_delay(dls, get_options());
+}
+
+/** Reset <b>dls</b> so that it will be considered downloadable
+ * immediately, and/or to show that we don't need it anymore.
+ *
+ * Must be called to initialise a download schedule, otherwise the zeroth item
+ * in the schedule will never be used.
+ *
+ * (We find the zeroth element of the download schedule, and set
+ * next_attempt_at to be the appropriate offset from 'now'. In most
+ * cases this means setting it to 'now', so the item will be immediately
+ * downloadable; when using authorities with fallbacks, there is a few seconds'
+ * delay.) */
+void
+download_status_reset(download_status_t *dls)
+{
+  if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD
+      || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD)
+    return; /* Don't reset this. */
+
+  dls->n_download_failures = 0;
+  dls->n_download_attempts = 0;
+  dls->next_attempt_at = download_status_get_initial_delay_from_now(dls);
+  dls->last_backoff_position = 0;
+  dls->last_delay_used = 0;
+  /* Don't reset dls->want_authority or dls->increment_on */
+}
+
+/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is
+ * ready to get its download reattempted. */
+int
+download_status_is_ready(download_status_t *dls, time_t now)
+{
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
+  return download_status_get_next_attempt_at(dls) <= now;
+}
+
+/** Mark <b>dl</b> as never downloadable. */
+void
+download_status_mark_impossible(download_status_t *dl)
+{
+  dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD;
+  dl->n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD;
+}
+
+/** Return the number of failures on <b>dls</b> since the last success (if
+ * any). */
+int
+download_status_get_n_failures(const download_status_t *dls)
+{
+  return dls->n_download_failures;
+}
+
+/** Return the number of attempts to download <b>dls</b> since the last success
+ * (if any). This can differ from download_status_get_n_failures() due to
+ * outstanding concurrent attempts. */
+int
+download_status_get_n_attempts(const download_status_t *dls)
+{
+  return dls->n_download_attempts;
+}
+
+/** Return the next time to attempt to download <b>dls</b>. */
+time_t
+download_status_get_next_attempt_at(const download_status_t *dls)
+{
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    /* so give the answer we would have given if it had been */
+    return download_status_get_initial_delay_from_now(dls);
+  }
+
+  return dls->next_attempt_at;
+}
+
+/** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>)
+ * fetches have failed (with uppercase fingerprints listed in <b>failed</b>,
+ * either as descriptor digests or as identity digests based on
+ * <b>was_descriptor_digests</b>).
+ */
+static void
+dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
+                               int router_purpose,
+                               int was_extrainfo, int was_descriptor_digests)
+{
+  char digest[DIGEST_LEN];
+  time_t now = time(NULL);
+  int server = directory_fetches_from_authorities(get_options());
+  if (!was_descriptor_digests) {
+    if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
+      tor_assert(!was_extrainfo);
+      connection_dir_retry_bridges(failed);
+    }
+    return; /* FFFF should implement for other-than-router-purpose someday */
+  }
+  SMARTLIST_FOREACH_BEGIN(failed, const char *, cp) {
+    download_status_t *dls = NULL;
+    if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
+      log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp));
+      continue;
+    }
+    if (was_extrainfo) {
+      signed_descriptor_t *sd =
+        router_get_by_extrainfo_digest(digest);
+      if (sd)
+        dls = &sd->ei_dl_status;
+    } else {
+      dls = router_get_dl_status_by_descriptor_digest(digest);
+    }
+    if (!dls)
+      continue;
+    download_status_increment_failure(dls, status_code, cp, server, now);
+  } SMARTLIST_FOREACH_END(cp);
+
+  /* No need to relaunch descriptor downloads here: we already do it
+   * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
+}
+
+/** Called when a connection to download microdescriptors from relay with
+ * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list
+ * of every microdesc digest we didn't get. <b>status_code</b> is the http
+ * status code we received. Reschedule the microdesc downloads as
+ * appropriate. */
+static void
+dir_microdesc_download_failed(smartlist_t *failed,
+                              int status_code, const char *dir_id)
+{
+  networkstatus_t *consensus
+    = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
+  routerstatus_t *rs;
+  download_status_t *dls;
+  time_t now = time(NULL);
+  int server = directory_fetches_from_authorities(get_options());
+
+  if (! consensus)
+    return;
+
+  /* We failed to fetch a microdescriptor from 'dir_id', note it down
+   * so that we don't try the same relay next time... */
+  microdesc_note_outdated_dirserver(dir_id);
+
+  SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
+    rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d);
+    if (!rs)
+      continue;
+    dls = &rs->dl_status;
+
+    { /* Increment the failure count for this md fetch */
+      char buf[BASE64_DIGEST256_LEN+1];
+      digest256_to_base64(buf, d);
+      log_info(LD_DIR, "Failed to download md %s from %s",
+               buf, hex_str(dir_id, DIGEST_LEN));
+      download_status_increment_failure(dls, status_code, buf,
+                                        server, now);
+    }
+  } SMARTLIST_FOREACH_END(d);
+}
+
+/** Helper.  Compare two fp_pair_t objects, and return negative, 0, or
+ * positive as appropriate. */
+static int
+compare_pairs_(const void **a, const void **b)
+{
+  const fp_pair_t *fp1 = *a, *fp2 = *b;
+  int r;
+  if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN)))
+    return r;
+  else
+    return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN);
+}
+
+/** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each
+ * FP is a hex-encoded fingerprint, into a sequence of distinct sorted
+ * fp_pair_t. Skip malformed pairs. On success, return 0 and add those
+ * fp_pair_t into <b>pairs_out</b>.  On failure, return -1. */
+int
+dir_split_resource_into_fingerprint_pairs(const char *res,
+                                          smartlist_t *pairs_out)
+{
+  smartlist_t *pairs_tmp = smartlist_new();
+  smartlist_t *pairs_result = smartlist_new();
+
+  smartlist_split_string(pairs_tmp, res, "+", 0, 0);
+  if (smartlist_len(pairs_tmp)) {
+    char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1);
+    size_t last_len = strlen(last);
+    if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
+      last[last_len-2] = '\0';
+    }
+  }
+  SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) {
+    if (strlen(cp) != HEX_DIGEST_LEN*2+1) {
+      log_info(LD_DIR,
+             "Skipping digest pair %s with non-standard length.", escaped(cp));
+    } else if (cp[HEX_DIGEST_LEN] != '-') {
+      log_info(LD_DIR,
+             "Skipping digest pair %s with missing dash.", escaped(cp));
+    } else {
+      fp_pair_t pair;
+      if (base16_decode(pair.first, DIGEST_LEN,
+                        cp, HEX_DIGEST_LEN) != DIGEST_LEN ||
+          base16_decode(pair.second,DIGEST_LEN,
+                        cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) {
+        log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp));
+      } else {
+        smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair)));
+      }
+    }
+    tor_free(cp);
+  } SMARTLIST_FOREACH_END(cp);
+  smartlist_free(pairs_tmp);
+
+  /* Uniq-and-sort */
+  smartlist_sort(pairs_result, compare_pairs_);
+  smartlist_uniq(pairs_result, compare_pairs_, tor_free_);
+
+  smartlist_add_all(pairs_out, pairs_result);
+  smartlist_free(pairs_result);
+  return 0;
+}
+
+/** Given a directory <b>resource</b> request, containing zero
+ * or more strings separated by plus signs, followed optionally by ".z", store
+ * the strings, in order, into <b>fp_out</b>.  If <b>compressed_out</b> is
+ * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
+ *
+ * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
+ * decode the rest.  If (flags & DSR_BASE64), then use "-" rather than "+" as
+ * a separator, delete all the elements that aren't base64-encoded digests,
+ * and decode the rest.  If (flags & DSR_DIGEST256), these digests should be
+ * 256 bits long; else they should be 160.
+ *
+ * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
+ */
+int
+dir_split_resource_into_fingerprints(const char *resource,
+                                     smartlist_t *fp_out, int *compressed_out,
+                                     int flags)
+{
+  const int decode_hex = flags & DSR_HEX;
+  const int decode_base64 = flags & DSR_BASE64;
+  const int digests_are_256 = flags & DSR_DIGEST256;
+  const int sort_uniq = flags & DSR_SORT_UNIQ;
+
+  const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
+  const int hex_digest_len = digests_are_256 ?
+    HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
+  const int base64_digest_len = digests_are_256 ?
+    BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
+  smartlist_t *fp_tmp = smartlist_new();
+
+  tor_assert(!(decode_hex && decode_base64));
+  tor_assert(fp_out);
+
+  smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
+  if (compressed_out)
+    *compressed_out = 0;
+  if (smartlist_len(fp_tmp)) {
+    char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1);
+    size_t last_len = strlen(last);
+    if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
+      last[last_len-2] = '\0';
+      if (compressed_out)
+        *compressed_out = 1;
+    }
+  }
+  if (decode_hex || decode_base64) {
+    const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
+    int i;
+    char *cp, *d = NULL;
+    for (i = 0; i < smartlist_len(fp_tmp); ++i) {
+      cp = smartlist_get(fp_tmp, i);
+      if (strlen(cp) != encoded_len) {
+        log_info(LD_DIR,
+                 "Skipping digest %s with non-standard length.", escaped(cp));
+        smartlist_del_keeporder(fp_tmp, i--);
+        goto again;
+      }
+      d = tor_malloc_zero(digest_len);
+      if (decode_hex ?
+          (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) :
+          (base64_decode(d, digest_len, cp, base64_digest_len)
+                         != digest_len)) {
+          log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
+          smartlist_del_keeporder(fp_tmp, i--);
+          goto again;
+      }
+      smartlist_set(fp_tmp, i, d);
+      d = NULL;
+    again:
+      tor_free(cp);
+      tor_free(d);
+    }
+  }
+  if (sort_uniq) {
+    if (decode_hex || decode_base64) {
+      if (digests_are_256) {
+        smartlist_sort_digests256(fp_tmp);
+        smartlist_uniq_digests256(fp_tmp);
+      } else {
+        smartlist_sort_digests(fp_tmp);
+        smartlist_uniq_digests(fp_tmp);
+      }
+    } else {
+      smartlist_sort_strings(fp_tmp);
+      smartlist_uniq_strings(fp_tmp);
+    }
+  }
+  smartlist_add_all(fp_out, fp_tmp);
+  smartlist_free(fp_tmp);
+  return 0;
+}
+
+/** As dir_split_resource_into_fingerprints, but instead fills
+ * <b>spool_out</b> with a list of spoolable_resource_t for the resource
+ * identified through <b>source</b>. */
+int
+dir_split_resource_into_spoolable(const char *resource,
+                                  dir_spool_source_t source,
+                                  smartlist_t *spool_out,
+                                  int *compressed_out,
+                                  int flags)
+{
+  smartlist_t *fingerprints = smartlist_new();
+
+  tor_assert(flags & (DSR_HEX|DSR_BASE64));
+  const size_t digest_len =
+    (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
+
+  int r = dir_split_resource_into_fingerprints(resource, fingerprints,
+                                               compressed_out, flags);
+  /* This is not a very efficient implementation XXXX */
+  SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
+    spooled_resource_t *spooled =
+      spooled_resource_new(source, digest, digest_len);
+    if (spooled)
+      smartlist_add(spool_out, spooled);
+    tor_free(digest);
+  } SMARTLIST_FOREACH_END(digest);
+
+  smartlist_free(fingerprints);
+  return r;
+}
diff --git a/src/feature/dircache/directory.h b/src/feature/dircache/directory.h
new file mode 100644
index 0000000000..992ff618fb
--- /dev/null
+++ b/src/feature/dircache/directory.h
@@ -0,0 +1,347 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file directory.h
+ * \brief Header file for directory.c.
+ **/
+
+#ifndef TOR_DIRECTORY_H
+#define TOR_DIRECTORY_H
+
+#include "or/hs_ident.h"
+enum compress_method_t;
+
+dir_connection_t *TO_DIR_CONN(connection_t *c);
+
+#define DIR_CONN_STATE_MIN_ 1
+/** State for connection to directory server: waiting for connect(). */
+#define DIR_CONN_STATE_CONNECTING 1
+/** State for connection to directory server: sending HTTP request. */
+#define DIR_CONN_STATE_CLIENT_SENDING 2
+/** State for connection to directory server: reading HTTP response. */
+#define DIR_CONN_STATE_CLIENT_READING 3
+/** State for connection to directory server: happy and finished. */
+#define DIR_CONN_STATE_CLIENT_FINISHED 4
+/** State for connection at directory server: waiting for HTTP request. */
+#define DIR_CONN_STATE_SERVER_COMMAND_WAIT 5
+/** State for connection at directory server: sending HTTP response. */
+#define DIR_CONN_STATE_SERVER_WRITING 6
+#define DIR_CONN_STATE_MAX_ 6
+
+#define DIR_PURPOSE_MIN_ 4
+/** A connection to a directory server: set after a v2 rendezvous
+ * descriptor is downloaded. */
+#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4
+/** A connection to a directory server: download one or more server
+ * descriptors. */
+#define DIR_PURPOSE_FETCH_SERVERDESC 6
+/** A connection to a directory server: download one or more extra-info
+ * documents. */
+#define DIR_PURPOSE_FETCH_EXTRAINFO 7
+/** A connection to a directory server: upload a server descriptor. */
+#define DIR_PURPOSE_UPLOAD_DIR 8
+/** A connection to a directory server: upload a v3 networkstatus vote. */
+#define DIR_PURPOSE_UPLOAD_VOTE 10
+/** A connection to a directory server: upload a v3 consensus signature */
+#define DIR_PURPOSE_UPLOAD_SIGNATURES 11
+/** A connection to a directory server: download one or more v3 networkstatus
+ * votes. */
+#define DIR_PURPOSE_FETCH_STATUS_VOTE 12
+/** A connection to a directory server: download a v3 detached signatures
+ * object for a consensus. */
+#define DIR_PURPOSE_FETCH_DETACHED_SIGNATURES 13
+/** A connection to a directory server: download a v3 networkstatus
+ * consensus. */
+#define DIR_PURPOSE_FETCH_CONSENSUS 14
+/** A connection to a directory server: download one or more directory
+ * authority certificates. */
+#define DIR_PURPOSE_FETCH_CERTIFICATE 15
+
+/** Purpose for connection at a directory server. */
+#define DIR_PURPOSE_SERVER 16
+/** A connection to a hidden service directory server: upload a v2 rendezvous
+ * descriptor. */
+#define DIR_PURPOSE_UPLOAD_RENDDESC_V2 17
+/** A connection to a hidden service directory server: download a v2 rendezvous
+ * descriptor. */
+#define DIR_PURPOSE_FETCH_RENDDESC_V2 18
+/** A connection to a directory server: download a microdescriptor. */
+#define DIR_PURPOSE_FETCH_MICRODESC 19
+/** A connection to a hidden service directory: upload a v3 descriptor. */
+#define DIR_PURPOSE_UPLOAD_HSDESC 20
+/** A connection to a hidden service directory: fetch a v3 descriptor. */
+#define DIR_PURPOSE_FETCH_HSDESC 21
+/** A connection to a directory server: set after a hidden service descriptor
+ * is downloaded. */
+#define DIR_PURPOSE_HAS_FETCHED_HSDESC 22
+#define DIR_PURPOSE_MAX_ 22
+
+/** True iff <b>p</b> is a purpose corresponding to uploading
+ * data to a directory server. */
+#define DIR_PURPOSE_IS_UPLOAD(p)                \
+  ((p)==DIR_PURPOSE_UPLOAD_DIR ||               \
+   (p)==DIR_PURPOSE_UPLOAD_VOTE ||              \
+   (p)==DIR_PURPOSE_UPLOAD_SIGNATURES ||        \
+   (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2 ||       \
+   (p)==DIR_PURPOSE_UPLOAD_HSDESC)
+
+int directories_have_accepted_server_descriptor(void);
+void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
+                                  dirinfo_type_t type, const char *payload,
+                                  size_t payload_len, size_t extrainfo_len);
+MOCK_DECL(void, directory_get_from_dirserver, (
+                          uint8_t dir_purpose,
+                          uint8_t router_purpose,
+                          const char *resource,
+                          int pds_flags,
+                          download_want_authority_t want_authority));
+void directory_get_from_all_authorities(uint8_t dir_purpose,
+                                        uint8_t router_purpose,
+                                        const char *resource);
+
+/** Enumeration of ways to connect to a directory server */
+typedef enum {
+  /** Default: connect over a one-hop Tor circuit. Relays fall back to direct
+   * DirPort connections, clients, onion services, and bridges do not */
+  DIRIND_ONEHOP=0,
+  /** Connect over a multi-hop anonymizing Tor circuit */
+  DIRIND_ANONYMOUS=1,
+  /** Connect to the DirPort directly */
+  DIRIND_DIRECT_CONN,
+  /** Connect over a multi-hop anonymizing Tor circuit to our dirport */
+  DIRIND_ANON_DIRPORT,
+} dir_indirection_t;
+
+int directory_must_use_begindir(const or_options_t *options);
+
+/**
+ * A directory_request_t describes the information about a directory request
+ * at the client side.  It describes what we're going to ask for, which
+ * directory we're going to ask for it, how we're going to contact that
+ * directory, and (in some cases) what to do with it when we're done.
+ */
+typedef struct directory_request_t directory_request_t;
+directory_request_t *directory_request_new(uint8_t dir_purpose);
+void directory_request_free_(directory_request_t *req);
+#define directory_request_free(req) \
+  FREE_AND_NULL(directory_request_t, directory_request_free_, (req))
+void directory_request_set_or_addr_port(directory_request_t *req,
+                                        const tor_addr_port_t *p);
+void directory_request_set_dir_addr_port(directory_request_t *req,
+                                         const tor_addr_port_t *p);
+void directory_request_set_directory_id_digest(directory_request_t *req,
+                                               const char *digest);
+struct circuit_guard_state_t;
+void directory_request_set_guard_state(directory_request_t *req,
+                                       struct circuit_guard_state_t *state);
+void directory_request_set_router_purpose(directory_request_t *req,
+                                          uint8_t router_purpose);
+void directory_request_set_indirection(directory_request_t *req,
+                                       dir_indirection_t indirection);
+void directory_request_set_resource(directory_request_t *req,
+                                    const char *resource);
+void directory_request_set_payload(directory_request_t *req,
+                                   const char *payload,
+                                   size_t payload_len);
+void directory_request_set_if_modified_since(directory_request_t *req,
+                                             time_t if_modified_since);
+void directory_request_set_rend_query(directory_request_t *req,
+                                      const rend_data_t *query);
+void directory_request_upload_set_hs_ident(directory_request_t *req,
+                                           const hs_ident_dir_conn_t *ident);
+void directory_request_fetch_set_hs_ident(directory_request_t *req,
+                                          const hs_ident_dir_conn_t *ident);
+
+void directory_request_set_routerstatus(directory_request_t *req,
+                                        const routerstatus_t *rs);
+void directory_request_add_header(directory_request_t *req,
+                                  const char *key,
+                                  const char *val);
+MOCK_DECL(void, directory_initiate_request, (directory_request_t *request));
+
+int parse_http_response(const char *headers, int *code, time_t *date,
+                        enum compress_method_t *compression, char **response);
+int parse_http_command(const char *headers,
+                       char **command_out, char **url_out);
+char *http_get_header(const char *headers, const char *which);
+
+int connection_dir_is_encrypted(const dir_connection_t *conn);
+int connection_dir_reached_eof(dir_connection_t *conn);
+int connection_dir_process_inbuf(dir_connection_t *conn);
+int connection_dir_finished_flushing(dir_connection_t *conn);
+int connection_dir_finished_connecting(dir_connection_t *conn);
+void connection_dir_about_to_close(dir_connection_t *dir_conn);
+
+#define DSR_HEX       (1<<0)
+#define DSR_BASE64    (1<<1)
+#define DSR_DIGEST256 (1<<2)
+#define DSR_SORT_UNIQ (1<<3)
+int dir_split_resource_into_fingerprints(const char *resource,
+                                     smartlist_t *fp_out, int *compressed_out,
+                                     int flags);
+enum dir_spool_source_t;
+int dir_split_resource_into_spoolable(const char *resource,
+                                      enum dir_spool_source_t source,
+                                      smartlist_t *spool_out,
+                                      int *compressed_out,
+                                      int flags);
+int dir_split_resource_into_fingerprint_pairs(const char *res,
+                                              smartlist_t *pairs_out);
+char *directory_dump_request_log(void);
+void note_request(const char *key, size_t bytes);
+int router_supports_extrainfo(const char *identity_digest, int is_authority);
+
+time_t download_status_increment_failure(download_status_t *dls,
+                                         int status_code, const char *item,
+                                         int server, time_t now);
+time_t download_status_increment_attempt(download_status_t *dls,
+                                         const char *item,  time_t now);
+/** Increment the failure count of the download_status_t <b>dls</b>, with
+ * the optional status code <b>sc</b>. */
+#define download_status_failed(dls, sc)                                 \
+  download_status_increment_failure((dls), (sc), NULL,                  \
+                                    dir_server_mode(get_options()), \
+                                    time(NULL))
+
+void download_status_reset(download_status_t *dls);
+int download_status_is_ready(download_status_t *dls, time_t now);
+time_t download_status_get_next_attempt_at(const download_status_t *dls);
+void download_status_mark_impossible(download_status_t *dl);
+
+int download_status_get_n_failures(const download_status_t *dls);
+int download_status_get_n_attempts(const download_status_t *dls);
+
+int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                            const char *resource);
+
+#ifdef DIRECTORY_PRIVATE
+
+/** A structure to hold arguments passed into each directory response
+ * handler */
+typedef struct response_handler_args_t {
+  int status_code;
+  const char *reason;
+  const char *body;
+  size_t body_len;
+  const char *headers;
+} response_handler_args_t;
+
+struct directory_request_t {
+  /**
+   * These fields specify which directory we're contacting.  Routerstatus,
+   * if present, overrides the other fields.
+   *
+   * @{ */
+  tor_addr_port_t or_addr_port;
+  tor_addr_port_t dir_addr_port;
+  char digest[DIGEST_LEN];
+
+  const routerstatus_t *routerstatus;
+  /** @} */
+  /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what
+   * kind of operation we'll be doing (upload/download), and of what kind
+   * of document. */
+  uint8_t dir_purpose;
+  /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo
+   * and extrainfo docs.  */
+  uint8_t router_purpose;
+  /** Enum: determines whether to anonymize, and whether to use dirport or
+   * orport. */
+  dir_indirection_t indirection;
+  /** Alias to the variable part of the URL for this request */
+  const char *resource;
+  /** Alias to the payload to upload (if any) */
+  const char *payload;
+  /** Number of bytes to upload from payload</b> */
+  size_t payload_len;
+  /** Value to send in an if-modified-since header, or 0 for none. */
+  time_t if_modified_since;
+  /** Hidden-service-specific information v2. */
+  const rend_data_t *rend_query;
+  /** Extra headers to append to the request */
+  struct config_line_t *additional_headers;
+  /** Hidden-service-specific information for v3+. */
+  const hs_ident_dir_conn_t *hs_ident;
+  /** Used internally to directory.c: gets informed when the attempt to
+   * connect to the directory succeeds or fails, if that attempt bears on the
+   * directory's usability as a directory guard. */
+  struct circuit_guard_state_t *guard_state;
+};
+
+struct get_handler_args_t;
+STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                                       const struct get_handler_args_t *args);
+STATIC int directory_handle_command(dir_connection_t *conn);
+STATIC char *accept_encoding_header(void);
+STATIC int allowed_anonymous_connection_compression_method(
+                                               enum compress_method_t);
+STATIC void warn_disallowed_anonymous_compression_method(
+                                               enum compress_method_t);
+
+STATIC int handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
+                                          const response_handler_args_t *args);
+STATIC int handle_response_fetch_microdesc(dir_connection_t *conn,
+                                 const response_handler_args_t *args);
+
+STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
+                                         const response_handler_args_t *args);
+
+#endif /* defined(DIRECTORY_PRIVATE) */
+
+#ifdef TOR_UNIT_TESTS
+/* Used only by test_dir.c and test_hs_cache.c */
+
+STATIC int parse_http_url(const char *headers, char **url);
+STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
+                                     const char *resource);
+MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn,
+                                                    const char *headers,
+                                                    const char *req_body,
+                                                    size_t req_body_len));
+MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn,
+                                                     const char *headers,
+                                                     const char *body,
+                                                     size_t body_len));
+STATIC int download_status_schedule_get_delay(download_status_t *dls,
+                                              int min_delay,
+                                              time_t now);
+
+STATIC int handle_post_hs_descriptor(const char *url, const char *body);
+
+STATIC char* authdir_type_to_string(dirinfo_type_t auth);
+STATIC const char * dir_conn_purpose_to_string(int purpose);
+STATIC int should_use_directory_guards(const or_options_t *options);
+enum compression_level_t;
+STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes);
+STATIC int find_dl_min_delay(const download_status_t *dls,
+                             const or_options_t *options);
+
+STATIC int next_random_exponential_delay(int delay,
+                                         int base_delay);
+
+STATIC void next_random_exponential_delay_range(int *low_bound_out,
+                                                int *high_bound_out,
+                                                int delay,
+                                                int base_delay);
+
+STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
+                                      const char **end_pos);
+
+STATIC unsigned parse_accept_encoding_header(const char *h);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#if defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE)
+/* Used only by directory.c and test_dir.c */
+
+/* no more than quadruple the previous delay (multiplier + 1) */
+#define DIR_DEFAULT_RANDOM_MULTIPLIER (3)
+/* no more than triple the previous delay */
+#define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
+
+#endif /* defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE) */
+
+#endif /* !defined(TOR_DIRECTORY_H) */
diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c
new file mode 100644
index 0000000000..2980d63f0a
--- /dev/null
+++ b/src/feature/dircache/dirserv.c
@@ -0,0 +1,3598 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DIRSERV_PRIVATE
+#include "or/or.h"
+#include "lib/container/buffers.h"
+#include "or/config.h"
+#include "or/confparse.h"
+#include "or/channel.h"
+#include "or/channeltls.h"
+#include "or/command.h"
+#include "or/connection.h"
+#include "or/connection_or.h"
+#include "or/conscache.h"
+#include "or/consdiffmgr.h"
+#include "or/control.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/hibernate.h"
+#include "or/keypin.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/protover.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+#include "or/torcert.h"
+#include "or/voting_schedule.h"
+
+#include "or/dirauth/dirvote.h"
+
+#include "or/cached_dir_st.h"
+#include "or/dir_connection_st.h"
+#include "or/extrainfo_st.h"
+#include "or/microdesc_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/tor_version_st.h"
+#include "or/vote_routerstatus_st.h"
+
+#include "lib/compress/compress.h"
+#include "lib/container/order.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/encoding/confline.h"
+
+/**
+ * \file dirserv.c
+ * \brief Directory server core implementation. Manages directory
+ * contents and generates directory documents.
+ *
+ * This module implements most of directory cache functionality, and some of
+ * the directory authority functionality.  The directory.c module delegates
+ * here in order to handle incoming requests from clients, via
+ * connection_dirserv_flushed_some() and its kin.  In order to save RAM, this
+ * module is responsible for spooling directory objects (in whole or in part)
+ * onto buf_t instances, and then closing the dir_connection_t once the
+ * objects are totally flushed.
+ *
+ * The directory.c module also delegates here for handling descriptor uploads
+ * via dirserv_add_multiple_descriptors().
+ *
+ * Additionally, this module handles some aspects of voting, including:
+ * deciding how to vote on individual flags (based on decisions reached in
+ * rephist.c), of formatting routerstatus lines, and deciding what relays to
+ * include in an authority's vote.  (TODO: Those functions could profitably be
+ * split off.  They only live in this file because historically they were
+ * shared among the v1, v2, and v3 directory code.)
+ */
+
+/** How far in the future do we allow a router to get? (seconds) */
+#define ROUTER_ALLOW_SKEW (60*60*12)
+/** How many seconds do we wait before regenerating the directory? */
+#define DIR_REGEN_SLACK_TIME 30
+/** If we're a cache, keep this many networkstatuses around from non-trusted
+ * directory authorities. */
+#define MAX_UNTRUSTED_NETWORKSTATUSES 16
+
+/** Total number of routers with measured bandwidth; this is set by
+ * dirserv_count_measured_bws() before the loop in
+ * dirserv_generate_networkstatus_vote_obj() and checked by
+ * dirserv_get_credible_bandwidth() and
+ * dirserv_compute_performance_thresholds() */
+static int routers_with_measured_bw = 0;
+
+static void directory_remove_invalid(void);
+struct authdir_config_t;
+static uint32_t
+dirserv_get_status_impl(const char *fp, const char *nickname,
+                        uint32_t addr, uint16_t or_port,
+                        const char *platform, const char **msg,
+                        int severity);
+static void clear_cached_dir(cached_dir_t *d);
+static const signed_descriptor_t *get_signed_descriptor_by_fp(
+                                                        const uint8_t *fp,
+                                                        int extrainfo);
+static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei,
+                                                const char **msg);
+static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri);
+
+static int spooled_resource_lookup_body(const spooled_resource_t *spooled,
+                                        int conn_is_encrypted,
+                                        const uint8_t **body_out,
+                                        size_t *size_out,
+                                        time_t *published_out);
+static cached_dir_t *spooled_resource_lookup_cached_dir(
+                                   const spooled_resource_t *spooled,
+                                   time_t *published_out);
+static cached_dir_t *lookup_cached_dir_by_fp(const uint8_t *fp);
+
+/************** Fingerprint handling code ************/
+
+/*                 1  Historically used to indicate Named */
+#define FP_INVALID 2  /**< Believed invalid. */
+#define FP_REJECT  4  /**< We will not publish this router. */
+/*                 8  Historically used to avoid using this as a dir. */
+#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */
+/*                 32 Historically used to indicade Unnamed */
+
+/** Target of status_by_digest map. */
+typedef uint32_t router_status_t;
+
+static void add_fingerprint_to_dir(const char *fp,
+                                   struct authdir_config_t *list,
+                                   router_status_t add_status);
+
+/** List of nickname-\>identity fingerprint mappings for all the routers
+ * that we name.  Used to prevent router impersonation. */
+typedef struct authdir_config_t {
+  strmap_t *fp_by_name; /**< Map from lc nickname to fingerprint. */
+  digestmap_t *status_by_digest; /**< Map from digest to router_status_t. */
+} authdir_config_t;
+
+/** Should be static; exposed for testing. */
+static authdir_config_t *fingerprint_list = NULL;
+
+/** Allocate and return a new, empty, authdir_config_t. */
+static authdir_config_t *
+authdir_config_new(void)
+{
+  authdir_config_t *list = tor_malloc_zero(sizeof(authdir_config_t));
+  list->fp_by_name = strmap_new();
+  list->status_by_digest = digestmap_new();
+  return list;
+}
+
+/** Add the fingerprint <b>fp</b> to the smartlist of fingerprint_entry_t's
+ * <b>list</b>, or-ing the currently set status flags with
+ * <b>add_status</b>.
+ */
+/* static */ void
+add_fingerprint_to_dir(const char *fp, authdir_config_t *list,
+                       router_status_t add_status)
+{
+  char *fingerprint;
+  char d[DIGEST_LEN];
+  router_status_t *status;
+  tor_assert(fp);
+  tor_assert(list);
+
+  fingerprint = tor_strdup(fp);
+  tor_strstrip(fingerprint, " ");
+  if (base16_decode(d, DIGEST_LEN,
+                    fingerprint, strlen(fingerprint)) != DIGEST_LEN) {
+    log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"",
+             escaped(fp));
+    tor_free(fingerprint);
+    return;
+  }
+
+  status = digestmap_get(list->status_by_digest, d);
+  if (!status) {
+    status = tor_malloc_zero(sizeof(router_status_t));
+    digestmap_set(list->status_by_digest, d, status);
+  }
+
+  tor_free(fingerprint);
+  *status |= add_status;
+  return;
+}
+
+/** Add the fingerprint for this OR to the global list of recognized
+ * identity key fingerprints. */
+int
+dirserv_add_own_fingerprint(crypto_pk_t *pk)
+{
+  char fp[FINGERPRINT_LEN+1];
+  if (crypto_pk_get_fingerprint(pk, fp, 0)<0) {
+    log_err(LD_BUG, "Error computing fingerprint");
+    return -1;
+  }
+  if (!fingerprint_list)
+    fingerprint_list = authdir_config_new();
+  add_fingerprint_to_dir(fp, fingerprint_list, 0);
+  return 0;
+}
+
+/** Load the nickname-\>fingerprint mappings stored in the approved-routers
+ * file.  The file format is line-based, with each non-blank holding one
+ * nickname, some space, and a fingerprint for that nickname.  On success,
+ * replace the current fingerprint list with the new list and return 0.  On
+ * failure, leave the current fingerprint list untouched, and return -1. */
+int
+dirserv_load_fingerprint_file(void)
+{
+  char *fname;
+  char *cf;
+  char *nickname, *fingerprint;
+  authdir_config_t *fingerprint_list_new;
+  int result;
+  config_line_t *front=NULL, *list;
+
+  fname = get_datadir_fname("approved-routers");
+  log_info(LD_GENERAL,
+           "Reloading approved fingerprints from \"%s\"...", fname);
+
+  cf = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+  if (!cf) {
+    log_warn(LD_FS, "Cannot open fingerprint file '%s'. That's ok.", fname);
+    tor_free(fname);
+    return 0;
+  }
+  tor_free(fname);
+
+  result = config_get_lines(cf, &front, 0);
+  tor_free(cf);
+  if (result < 0) {
+    log_warn(LD_CONFIG, "Error reading from fingerprint file");
+    return -1;
+  }
+
+  fingerprint_list_new = authdir_config_new();
+
+  for (list=front; list; list=list->next) {
+    char digest_tmp[DIGEST_LEN];
+    router_status_t add_status = 0;
+    nickname = list->key; fingerprint = list->value;
+    tor_strstrip(fingerprint, " "); /* remove spaces */
+    if (strlen(fingerprint) != HEX_DIGEST_LEN ||
+        base16_decode(digest_tmp, sizeof(digest_tmp),
+                      fingerprint, HEX_DIGEST_LEN) != sizeof(digest_tmp)) {
+      log_notice(LD_CONFIG,
+                 "Invalid fingerprint (nickname '%s', "
+                 "fingerprint %s). Skipping.",
+                 nickname, fingerprint);
+      continue;
+    }
+    if (!strcasecmp(nickname, "!reject")) {
+        add_status = FP_REJECT;
+    } else if (!strcasecmp(nickname, "!badexit")) {
+        add_status = FP_BADEXIT;
+    } else if (!strcasecmp(nickname, "!invalid")) {
+        add_status = FP_INVALID;
+    }
+    add_fingerprint_to_dir(fingerprint, fingerprint_list_new, add_status);
+  }
+
+  config_free_lines(front);
+  dirserv_free_fingerprint_list();
+  fingerprint_list = fingerprint_list_new;
+  /* Delete any routers whose fingerprints we no longer recognize */
+  directory_remove_invalid();
+  return 0;
+}
+
+/* If this is set, then we don't allow routers that have advertised an Ed25519
+ * identity to stop doing so.  This is going to be essential for good identity
+ * security: otherwise anybody who can attack RSA-1024 but not Ed25519 could
+ * just sign fake descriptors missing the Ed25519 key.  But we won't actually
+ * be able to prevent that kind of thing until we're confident that there isn't
+ * actually a legit reason to downgrade to 0.2.5.  Now we are not recommending
+ * 0.2.5 anymore so there is no reason to keep the #undef.
+ */
+
+#define DISABLE_DISABLING_ED25519
+
+/** Check whether <b>router</b> has a nickname/identity key combination that
+ * we recognize from the fingerprint list, or an IP we automatically act on
+ * according to our configuration.  Return the appropriate router status.
+ *
+ * If the status is 'FP_REJECT' and <b>msg</b> is provided, set
+ * *<b>msg</b> to an explanation of why. */
+uint32_t
+dirserv_router_get_status(const routerinfo_t *router, const char **msg,
+                          int severity)
+{
+  char d[DIGEST_LEN];
+  const int key_pinning = get_options()->AuthDirPinKeys;
+
+  if (crypto_pk_get_digest(router->identity_pkey, d)) {
+    log_warn(LD_BUG,"Error computing fingerprint");
+    if (msg)
+      *msg = "Bug: Error computing fingerprint";
+    return FP_REJECT;
+  }
+
+  /* Check for the more usual versions to reject a router first. */
+  const uint32_t r = dirserv_get_status_impl(d, router->nickname,
+                                             router->addr, router->or_port,
+                                             router->platform, msg, severity);
+  if (r)
+    return r;
+
+  /* dirserv_get_status_impl already rejects versions older than 0.2.4.18-rc,
+   * and onion_curve25519_pkey was introduced in 0.2.4.8-alpha.
+   * But just in case a relay doesn't provide or lies about its version, or
+   * doesn't include an ntor key in its descriptor, check that it exists,
+   * and is non-zero (clients check that it's non-zero before using it). */
+  if (!routerinfo_has_curve25519_onion_key(router)) {
+    log_fn(severity, LD_DIR,
+           "Descriptor from router %s is missing an ntor curve25519 onion "
+           "key.", router_describe(router));
+    if (msg)
+      *msg = "Missing ntor curve25519 onion key. Please upgrade!";
+    return FP_REJECT;
+  }
+
+  if (router->cache_info.signing_key_cert) {
+    /* This has an ed25519 identity key. */
+    if (KEYPIN_MISMATCH ==
+        keypin_check((const uint8_t*)router->cache_info.identity_digest,
+                   router->cache_info.signing_key_cert->signing_key.pubkey)) {
+      log_fn(severity, LD_DIR,
+             "Descriptor from router %s has an Ed25519 key, "
+               "but the <rsa,ed25519> keys don't match what they were before.",
+               router_describe(router));
+      if (key_pinning) {
+        if (msg) {
+          *msg = "Ed25519 identity key or RSA identity key has changed.";
+        }
+        return FP_REJECT;
+      }
+    }
+  } else {
+    /* No ed25519 key */
+    if (KEYPIN_MISMATCH == keypin_check_lone_rsa(
+                        (const uint8_t*)router->cache_info.identity_digest)) {
+      log_fn(severity, LD_DIR,
+               "Descriptor from router %s has no Ed25519 key, "
+               "when we previously knew an Ed25519 for it. Ignoring for now, "
+               "since Ed25519 keys are fairly new.",
+               router_describe(router));
+#ifdef DISABLE_DISABLING_ED25519
+      if (key_pinning) {
+        if (msg) {
+          *msg = "Ed25519 identity key has disappeared.";
+        }
+        return FP_REJECT;
+      }
+#endif /* defined(DISABLE_DISABLING_ED25519) */
+    }
+  }
+
+  return 0;
+}
+
+/** Return true if there is no point in downloading the router described by
+ * <b>rs</b> because this directory would reject it. */
+int
+dirserv_would_reject_router(const routerstatus_t *rs)
+{
+  uint32_t res;
+
+  res = dirserv_get_status_impl(rs->identity_digest, rs->nickname,
+                                rs->addr, rs->or_port,
+                                NULL, NULL, LOG_DEBUG);
+
+  return (res & FP_REJECT) != 0;
+}
+
+/** Helper: As dirserv_router_get_status, but takes the router fingerprint
+ * (hex, no spaces), nickname, address (used for logging only), IP address, OR
+ * port and platform (logging only) as arguments.
+ *
+ * Log messages at 'severity'. (There's not much point in
+ * logging that we're rejecting servers we'll not download.)
+ */
+static uint32_t
+dirserv_get_status_impl(const char *id_digest, const char *nickname,
+                        uint32_t addr, uint16_t or_port,
+                        const char *platform, const char **msg, int severity)
+{
+  uint32_t result = 0;
+  router_status_t *status_by_digest;
+
+  if (!fingerprint_list)
+    fingerprint_list = authdir_config_new();
+
+  log_debug(LD_DIRSERV, "%d fingerprints, %d digests known.",
+            strmap_size(fingerprint_list->fp_by_name),
+            digestmap_size(fingerprint_list->status_by_digest));
+
+  if (platform) {
+    tor_version_t ver_tmp;
+    if (tor_version_parse_platform(platform, &ver_tmp, 1) < 0) {
+      if (msg) {
+        *msg = "Malformed platform string.";
+      }
+      return FP_REJECT;
+    }
+  }
+
+  /* Versions before Tor 0.2.4.18-rc are too old to support, and are
+   * missing some important security fixes too. Disable them. */
+  if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) {
+    if (msg)
+      *msg = "Tor version is insecure or unsupported. Please upgrade!";
+    return FP_REJECT;
+  }
+
+  /* Tor 0.2.9.x where x<5 suffers from bug #20499, where relays don't
+   * keep their consensus up to date so they make bad guards.
+   * The simple fix is to just drop them from the network. */
+  if (platform &&
+      tor_version_as_new_as(platform,"0.2.9.0-alpha") &&
+      !tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
+    if (msg)
+      *msg = "Tor version contains bug 20499. Please upgrade!";
+    return FP_REJECT;
+  }
+
+  status_by_digest = digestmap_get(fingerprint_list->status_by_digest,
+                                   id_digest);
+  if (status_by_digest)
+    result |= *status_by_digest;
+
+  if (result & FP_REJECT) {
+    if (msg)
+      *msg = "Fingerprint is marked rejected -- if you think this is a "
+             "mistake please set a valid email address in ContactInfo and "
+             "send an email to bad-relays@lists.torproject.org mentioning "
+             "your fingerprint(s)?";
+    return FP_REJECT;
+  } else if (result & FP_INVALID) {
+    if (msg)
+      *msg = "Fingerprint is marked invalid";
+  }
+
+  if (authdir_policy_badexit_address(addr, or_port)) {
+    log_fn(severity, LD_DIRSERV,
+           "Marking '%s' as bad exit because of address '%s'",
+               nickname, fmt_addr32(addr));
+    result |= FP_BADEXIT;
+  }
+
+  if (!authdir_policy_permits_address(addr, or_port)) {
+    log_fn(severity, LD_DIRSERV, "Rejecting '%s' because of address '%s'",
+               nickname, fmt_addr32(addr));
+    if (msg)
+      *msg = "Suspicious relay address range -- if you think this is a "
+             "mistake please set a valid email address in ContactInfo and "
+             "send an email to bad-relays@lists.torproject.org mentioning "
+             "your address(es) and fingerprint(s)?";
+    return FP_REJECT;
+  }
+  if (!authdir_policy_valid_address(addr, or_port)) {
+    log_fn(severity, LD_DIRSERV,
+           "Not marking '%s' valid because of address '%s'",
+               nickname, fmt_addr32(addr));
+    result |= FP_INVALID;
+  }
+
+  return result;
+}
+
+/** Clear the current fingerprint list. */
+void
+dirserv_free_fingerprint_list(void)
+{
+  if (!fingerprint_list)
+    return;
+
+  strmap_free(fingerprint_list->fp_by_name, tor_free_);
+  digestmap_free(fingerprint_list->status_by_digest, tor_free_);
+  tor_free(fingerprint_list);
+}
+
+/*
+ *    Descriptor list
+ */
+
+/** Return -1 if <b>ri</b> has a private or otherwise bad address,
+ * unless we're configured to not care. Return 0 if all ok. */
+static int
+dirserv_router_has_valid_address(routerinfo_t *ri)
+{
+  tor_addr_t addr;
+  if (get_options()->DirAllowPrivateAddresses)
+    return 0; /* whatever it is, we're fine with it */
+  tor_addr_from_ipv4h(&addr, ri->addr);
+
+  if (tor_addr_is_internal(&addr, 0)) {
+    log_info(LD_DIRSERV,
+             "Router %s published internal IP address. Refusing.",
+             router_describe(ri));
+    return -1; /* it's a private IP, we should reject it */
+  }
+  return 0;
+}
+
+/** Check whether we, as a directory server, want to accept <b>ri</b>.  If so,
+ * set its is_valid,running fields and return 0.  Otherwise, return -1.
+ *
+ * If the router is rejected, set *<b>msg</b> to an explanation of why.
+ *
+ * If <b>complain</b> then explain at log-level 'notice' why we refused
+ * a descriptor; else explain at log-level 'info'.
+ */
+int
+authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                               int complain, int *valid_out)
+{
+  /* Okay.  Now check whether the fingerprint is recognized. */
+  time_t now;
+  int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO;
+  uint32_t status = dirserv_router_get_status(ri, msg, severity);
+  tor_assert(msg);
+  if (status & FP_REJECT)
+    return -1; /* msg is already set. */
+
+  /* Is there too much clock skew? */
+  now = time(NULL);
+  if (ri->cache_info.published_on > now+ROUTER_ALLOW_SKEW) {
+    log_fn(severity, LD_DIRSERV, "Publication time for %s is too "
+           "far (%d minutes) in the future; possible clock skew. Not adding "
+           "(%s)",
+           router_describe(ri),
+           (int)((ri->cache_info.published_on-now)/60),
+           esc_router_info(ri));
+    *msg = "Rejected: Your clock is set too far in the future, or your "
+      "timezone is not correct.";
+    return -1;
+  }
+  if (ri->cache_info.published_on < now-ROUTER_MAX_AGE_TO_PUBLISH) {
+    log_fn(severity, LD_DIRSERV,
+           "Publication time for %s is too far "
+           "(%d minutes) in the past. Not adding (%s)",
+           router_describe(ri),
+           (int)((now-ri->cache_info.published_on)/60),
+           esc_router_info(ri));
+    *msg = "Rejected: Server is expired, or your clock is too far in the past,"
+      " or your timezone is not correct.";
+    return -1;
+  }
+  if (dirserv_router_has_valid_address(ri) < 0) {
+    log_fn(severity, LD_DIRSERV,
+           "Router %s has invalid address. Not adding (%s).",
+           router_describe(ri),
+           esc_router_info(ri));
+    *msg = "Rejected: Address is a private address.";
+    return -1;
+  }
+
+  *valid_out = ! (status & FP_INVALID);
+
+  return 0;
+}
+
+/** Update the relevant flags of <b>node</b> based on our opinion as a
+ * directory authority in <b>authstatus</b>, as returned by
+ * dirserv_router_get_status or equivalent.  */
+void
+dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                 uint32_t authstatus)
+{
+  node->is_valid = (authstatus & FP_INVALID) ? 0 : 1;
+  node->is_bad_exit = (authstatus & FP_BADEXIT) ? 1 : 0;
+}
+
+/** True iff <b>a</b> is more severe than <b>b</b>. */
+static int
+WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b)
+{
+  return a < b;
+}
+
+/** As for dirserv_add_descriptor(), but accepts multiple documents, and
+ * returns the most severe error that occurred for any one of them. */
+was_router_added_t
+dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
+                                 const char *source,
+                                 const char **msg)
+{
+  was_router_added_t r, r_tmp;
+  const char *msg_out;
+  smartlist_t *list;
+  const char *s;
+  int n_parsed = 0;
+  time_t now = time(NULL);
+  char annotation_buf[ROUTER_ANNOTATION_BUF_LEN];
+  char time_buf[ISO_TIME_LEN+1];
+  int general = purpose == ROUTER_PURPOSE_GENERAL;
+  tor_assert(msg);
+
+  r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */
+
+  format_iso_time(time_buf, now);
+  if (tor_snprintf(annotation_buf, sizeof(annotation_buf),
+                   "@uploaded-at %s\n"
+                   "@source %s\n"
+                   "%s%s%s", time_buf, escaped(source),
+                   !general ? "@purpose " : "",
+                   !general ? router_purpose_to_string(purpose) : "",
+                   !general ? "\n" : "")<0) {
+    *msg = "Couldn't format annotations";
+    /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is
+     * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */
+    return -1;
+  }
+
+  s = desc;
+  list = smartlist_new();
+  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0,
+                                     annotation_buf, NULL)) {
+    SMARTLIST_FOREACH(list, routerinfo_t *, ri, {
+        msg_out = NULL;
+        tor_assert(ri->purpose == purpose);
+        r_tmp = dirserv_add_descriptor(ri, &msg_out, source);
+        if (WRA_MORE_SEVERE(r_tmp, r)) {
+          r = r_tmp;
+          *msg = msg_out;
+        }
+      });
+  }
+  n_parsed += smartlist_len(list);
+  smartlist_clear(list);
+
+  s = desc;
+  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0,
+                                     NULL, NULL)) {
+    SMARTLIST_FOREACH(list, extrainfo_t *, ei, {
+        msg_out = NULL;
+
+        r_tmp = dirserv_add_extrainfo(ei, &msg_out);
+        if (WRA_MORE_SEVERE(r_tmp, r)) {
+          r = r_tmp;
+          *msg = msg_out;
+        }
+      });
+  }
+  n_parsed += smartlist_len(list);
+  smartlist_free(list);
+
+  if (! *msg) {
+    if (!n_parsed) {
+      *msg = "No descriptors found in your POST.";
+      if (WRA_WAS_ADDED(r))
+        r = ROUTER_IS_ALREADY_KNOWN;
+    } else {
+      *msg = "(no message)";
+    }
+  }
+
+  return r;
+}
+
+/** Examine the parsed server descriptor in <b>ri</b> and maybe insert it into
+ * the list of server descriptors. Set *<b>msg</b> to a message that should be
+ * passed back to the origin of this descriptor, or NULL if there is no such
+ * message. Use <b>source</b> to produce better log messages.
+ *
+ * If <b>ri</b> is not added to the list of server descriptors, free it.
+ * That means the caller must not access <b>ri</b> after this function
+ * returns, since it might have been freed.
+ *
+ * Return the status of the operation.
+ *
+ * This function is only called when fresh descriptors are posted, not when
+ * we re-load the cache.
+ */
+was_router_added_t
+dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
+{
+  was_router_added_t r;
+  routerinfo_t *ri_old;
+  char *desc, *nickname;
+  const size_t desclen = ri->cache_info.signed_descriptor_len +
+      ri->cache_info.annotations_len;
+  const int key_pinning = get_options()->AuthDirPinKeys;
+  *msg = NULL;
+
+  /* If it's too big, refuse it now. Otherwise we'll cache it all over the
+   * network and it'll clog everything up. */
+  if (ri->cache_info.signed_descriptor_len > MAX_DESCRIPTOR_UPLOAD_SIZE) {
+    log_notice(LD_DIR, "Somebody attempted to publish a router descriptor '%s'"
+               " (source: %s) with size %d. Either this is an attack, or the "
+               "MAX_DESCRIPTOR_UPLOAD_SIZE (%d) constant is too low.",
+               ri->nickname, source, (int)ri->cache_info.signed_descriptor_len,
+               MAX_DESCRIPTOR_UPLOAD_SIZE);
+    *msg = "Router descriptor was too large.";
+    r = ROUTER_AUTHDIR_REJECTS;
+    goto fail;
+  }
+
+  /* Check whether this descriptor is semantically identical to the last one
+   * from this server.  (We do this here and not in router_add_to_routerlist
+   * because we want to be able to accept the newest router descriptor that
+   * another authority has, so we all converge on the same one.) */
+  ri_old = router_get_mutable_by_digest(ri->cache_info.identity_digest);
+  if (ri_old && ri_old->cache_info.published_on < ri->cache_info.published_on
+      && router_differences_are_cosmetic(ri_old, ri)
+      && !router_is_me(ri)) {
+    log_info(LD_DIRSERV,
+             "Not replacing descriptor from %s (source: %s); "
+             "differences are cosmetic.",
+             router_describe(ri), source);
+    *msg = "Not replacing router descriptor; no information has changed since "
+      "the last one with this identity.";
+    r = ROUTER_IS_ALREADY_KNOWN;
+    goto fail;
+  }
+
+  /* Do keypinning again ... this time, to add the pin if appropriate */
+  int keypin_status;
+  if (ri->cache_info.signing_key_cert) {
+    ed25519_public_key_t *pkey = &ri->cache_info.signing_key_cert->signing_key;
+    /* First let's validate this pubkey before pinning it */
+    if (ed25519_validate_pubkey(pkey) < 0) {
+      log_warn(LD_DIRSERV, "Received bad key from %s (source %s)",
+               router_describe(ri), source);
+      routerinfo_free(ri);
+      return ROUTER_AUTHDIR_REJECTS;
+    }
+
+    /* Now pin it! */
+    keypin_status = keypin_check_and_add(
+      (const uint8_t*)ri->cache_info.identity_digest,
+      pkey->pubkey, ! key_pinning);
+  } else {
+    keypin_status = keypin_check_lone_rsa(
+      (const uint8_t*)ri->cache_info.identity_digest);
+#ifndef DISABLE_DISABLING_ED25519
+    if (keypin_status == KEYPIN_MISMATCH)
+      keypin_status = KEYPIN_NOT_FOUND;
+#endif
+  }
+  if (keypin_status == KEYPIN_MISMATCH && key_pinning) {
+    log_info(LD_DIRSERV, "Dropping descriptor from %s (source: %s) because "
+             "its key did not match an older RSA/Ed25519 keypair",
+             router_describe(ri), source);
+    *msg = "Looks like your keypair has changed? This authority previously "
+      "recorded a different RSA identity for this Ed25519 identity (or vice "
+      "versa.) Did you replace or copy some of your key files, but not "
+      "the others? You should either restore the expected keypair, or "
+      "delete your keys and restart Tor to start your relay with a new "
+      "identity.";
+    r = ROUTER_AUTHDIR_REJECTS;
+    goto fail;
+  }
+
+  /* Make a copy of desc, since router_add_to_routerlist might free
+   * ri and its associated signed_descriptor_t. */
+  desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen);
+  nickname = tor_strdup(ri->nickname);
+
+  /* Tell if we're about to need to launch a test if we add this. */
+  ri->needs_retest_if_added =
+    dirserv_should_launch_reachability_test(ri, ri_old);
+
+  r = router_add_to_routerlist(ri, msg, 0, 0);
+  if (!WRA_WAS_ADDED(r)) {
+    /* unless the routerinfo was fine, just out-of-date */
+    log_info(LD_DIRSERV,
+             "Did not add descriptor from '%s' (source: %s): %s.",
+             nickname, source, *msg ? *msg : "(no message)");
+  } else {
+    smartlist_t *changed;
+
+    changed = smartlist_new();
+    smartlist_add(changed, ri);
+    routerlist_descriptors_added(changed, 0);
+    smartlist_free(changed);
+    if (!*msg) {
+      *msg =  "Descriptor accepted";
+    }
+    log_info(LD_DIRSERV,
+             "Added descriptor from '%s' (source: %s): %s.",
+             nickname, source, *msg);
+  }
+  tor_free(desc);
+  tor_free(nickname);
+  return r;
+ fail:
+  {
+    const char *desc_digest = ri->cache_info.signed_descriptor_digest;
+    download_status_t *dls =
+      router_get_dl_status_by_descriptor_digest(desc_digest);
+    if (dls) {
+      log_info(LD_GENERAL, "Marking router with descriptor %s as rejected, "
+               "and therefore undownloadable",
+               hex_str(desc_digest, DIGEST_LEN));
+      download_status_mark_impossible(dls);
+    }
+    routerinfo_free(ri);
+  }
+  return r;
+}
+
+/** As dirserv_add_descriptor, but for an extrainfo_t <b>ei</b>. */
+static was_router_added_t
+dirserv_add_extrainfo(extrainfo_t *ei, const char **msg)
+{
+  routerinfo_t *ri;
+  int r;
+  was_router_added_t rv;
+  tor_assert(msg);
+  *msg = NULL;
+
+  /* Needs to be mutable so routerinfo_incompatible_with_extrainfo
+   * can mess with some of the flags in ri->cache_info. */
+  ri = router_get_mutable_by_digest(ei->cache_info.identity_digest);
+  if (!ri) {
+    *msg = "No corresponding router descriptor for extra-info descriptor";
+    rv = ROUTER_BAD_EI;
+    goto fail;
+  }
+
+  /* If it's too big, refuse it now. Otherwise we'll cache it all over the
+   * network and it'll clog everything up. */
+  if (ei->cache_info.signed_descriptor_len > MAX_EXTRAINFO_UPLOAD_SIZE) {
+    log_notice(LD_DIR, "Somebody attempted to publish an extrainfo "
+               "with size %d. Either this is an attack, or the "
+               "MAX_EXTRAINFO_UPLOAD_SIZE (%d) constant is too low.",
+               (int)ei->cache_info.signed_descriptor_len,
+               MAX_EXTRAINFO_UPLOAD_SIZE);
+    *msg = "Extrainfo document was too large";
+    rv = ROUTER_BAD_EI;
+    goto fail;
+  }
+
+  if ((r = routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
+                                                  &ri->cache_info, msg))) {
+    if (r<0) {
+      extrainfo_free(ei);
+      return ROUTER_IS_ALREADY_KNOWN;
+    }
+    rv = ROUTER_BAD_EI;
+    goto fail;
+  }
+  router_add_extrainfo_to_routerlist(ei, msg, 0, 0);
+  return ROUTER_ADDED_SUCCESSFULLY;
+ fail:
+  {
+    const char *d = ei->cache_info.signed_descriptor_digest;
+    signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d);
+    if (sd) {
+      log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as "
+               "rejected, and therefore undownloadable",
+               hex_str((char*)d,DIGEST_LEN));
+      download_status_mark_impossible(&sd->ei_dl_status);
+    }
+    extrainfo_free(ei);
+  }
+  return rv;
+}
+
+/** Remove all descriptors whose nicknames or fingerprints no longer
+ * are allowed by our fingerprint list. (Descriptors that used to be
+ * good can become bad when we reload the fingerprint list.)
+ */
+static void
+directory_remove_invalid(void)
+{
+  routerlist_t *rl = router_get_routerlist();
+  smartlist_t *nodes = smartlist_new();
+  smartlist_add_all(nodes, nodelist_get_list());
+
+  SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) {
+    const char *msg = NULL;
+    const char *description;
+    routerinfo_t *ent = node->ri;
+    uint32_t r;
+    if (!ent)
+      continue;
+    r = dirserv_router_get_status(ent, &msg, LOG_INFO);
+    description = router_describe(ent);
+    if (r & FP_REJECT) {
+      log_info(LD_DIRSERV, "Router %s is now rejected: %s",
+               description, msg?msg:"");
+      routerlist_remove(rl, ent, 0, time(NULL));
+      continue;
+    }
+    if (bool_neq((r & FP_INVALID), !node->is_valid)) {
+      log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description,
+               (r&FP_INVALID) ? "in" : "");
+      node->is_valid = (r&FP_INVALID)?0:1;
+    }
+    if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) {
+      log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description,
+               (r & FP_BADEXIT) ? "bad" : "good");
+      node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0;
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  routerlist_assert_ok(rl);
+  smartlist_free(nodes);
+}
+
+/**
+ * Allocate and return a description of the status of the server <b>desc</b>,
+ * for use in a v1-style router-status line.  The server is listed
+ * as running iff <b>is_live</b> is true.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
+ */
+static char *
+list_single_server_status(const routerinfo_t *desc, int is_live)
+{
+  char buf[MAX_NICKNAME_LEN+HEX_DIGEST_LEN+4]; /* !nickname=$hexdigest\0 */
+  char *cp;
+  const node_t *node;
+
+  tor_assert(desc);
+
+  cp = buf;
+  if (!is_live) {
+    *cp++ = '!';
+  }
+  node = node_get_by_id(desc->cache_info.identity_digest);
+  if (node && node->is_valid) {
+    strlcpy(cp, desc->nickname, sizeof(buf)-(cp-buf));
+    cp += strlen(cp);
+    *cp++ = '=';
+  }
+  *cp++ = '$';
+  base16_encode(cp, HEX_DIGEST_LEN+1, desc->cache_info.identity_digest,
+                DIGEST_LEN);
+  return tor_strdup(buf);
+}
+
+/* DOCDOC running_long_enough_to_decide_unreachable */
+int
+running_long_enough_to_decide_unreachable(void)
+{
+  return time_of_process_start
+    + get_options()->TestingAuthDirTimeToLearnReachability < approx_time();
+}
+
+/** Each server needs to have passed a reachability test no more
+ * than this number of seconds ago, or it is listed as down in
+ * the directory. */
+#define REACHABLE_TIMEOUT (45*60)
+
+/** If we tested a router and found it reachable _at least this long_ after it
+ * declared itself hibernating, it is probably done hibernating and we just
+ * missed a descriptor from it. */
+#define HIBERNATION_PUBLICATION_SKEW (60*60)
+
+/** Treat a router as alive if
+ *    - It's me, and I'm not hibernating.
+ * or - We've found it reachable recently. */
+void
+dirserv_set_router_is_running(routerinfo_t *router, time_t now)
+{
+  /*XXXX This function is a mess.  Separate out the part that calculates
+    whether it's reachable and the part that tells rephist that the router was
+    unreachable.
+   */
+  int answer;
+  const or_options_t *options = get_options();
+  node_t *node = node_get_mutable_by_id(router->cache_info.identity_digest);
+  tor_assert(node);
+
+  if (router_is_me(router)) {
+    /* We always know if we are shutting down or hibernating ourselves. */
+    answer = ! we_are_hibernating();
+  } else if (router->is_hibernating &&
+             (router->cache_info.published_on +
+              HIBERNATION_PUBLICATION_SKEW) > node->last_reachable) {
+    /* A hibernating router is down unless we (somehow) had contact with it
+     * since it declared itself to be hibernating. */
+    answer = 0;
+  } else if (options->AssumeReachable) {
+    /* If AssumeReachable, everybody is up unless they say they are down! */
+    answer = 1;
+  } else {
+    /* Otherwise, a router counts as up if we found all announced OR
+       ports reachable in the last REACHABLE_TIMEOUT seconds.
+
+       XXX prop186 For now there's always one IPv4 and at most one
+       IPv6 OR port.
+
+       If we're not on IPv6, don't consider reachability of potential
+       IPv6 OR port since that'd kill all dual stack relays until a
+       majority of the dir auths have IPv6 connectivity. */
+    answer = (now < node->last_reachable + REACHABLE_TIMEOUT &&
+              (options->AuthDirHasIPv6Connectivity != 1 ||
+               tor_addr_is_null(&router->ipv6_addr) ||
+               now < node->last_reachable6 + REACHABLE_TIMEOUT));
+  }
+
+  if (!answer && running_long_enough_to_decide_unreachable()) {
+    /* Not considered reachable. tell rephist about that.
+
+       Because we launch a reachability test for each router every
+       REACHABILITY_TEST_CYCLE_PERIOD seconds, then the router has probably
+       been down since at least that time after we last successfully reached
+       it.
+
+       XXX ipv6
+     */
+    time_t when = now;
+    if (node->last_reachable &&
+        node->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD < now)
+      when = node->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD;
+    rep_hist_note_router_unreachable(router->cache_info.identity_digest, when);
+  }
+
+  node->is_running = answer;
+}
+
+/** Based on the routerinfo_ts in <b>routers</b>, allocate the
+ * contents of a v1-style router-status line, and store it in
+ * *<b>router_status_out</b>.  Return 0 on success, -1 on failure.
+ *
+ * If for_controller is true, include the routers with very old descriptors.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
+ */
+int
+list_server_status_v1(smartlist_t *routers, char **router_status_out,
+                      int for_controller)
+{
+  /* List of entries in a router-status style: An optional !, then an optional
+   * equals-suffixed nickname, then a dollar-prefixed hexdigest. */
+  smartlist_t *rs_entries;
+  time_t now = time(NULL);
+  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  const or_options_t *options = get_options();
+  /* We include v2 dir auths here too, because they need to answer
+   * controllers. Eventually we'll deprecate this whole function;
+   * see also networkstatus_getinfo_by_purpose(). */
+  int authdir = authdir_mode_publishes_statuses(options);
+  tor_assert(router_status_out);
+
+  rs_entries = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
+    tor_assert(node);
+    if (authdir) {
+      /* Update router status in routerinfo_t. */
+      dirserv_set_router_is_running(ri, now);
+    }
+    if (for_controller) {
+      char name_buf[MAX_VERBOSE_NICKNAME_LEN+2];
+      char *cp = name_buf;
+      if (!node->is_running)
+        *cp++ = '!';
+      router_get_verbose_nickname(cp, ri);
+      smartlist_add_strdup(rs_entries, name_buf);
+    } else if (ri->cache_info.published_on >= cutoff) {
+      smartlist_add(rs_entries, list_single_server_status(ri,
+                                                          node->is_running));
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(rs_entries, char *, cp, tor_free(cp));
+  smartlist_free(rs_entries);
+
+  return 0;
+}
+
+/** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid,
+ * not hibernating, having observed bw greater 0, and not too old. Else
+ * return 0.
+ */
+static int
+router_is_active(const routerinfo_t *ri, const node_t *node, time_t now)
+{
+  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  if (ri->cache_info.published_on < cutoff) {
+    return 0;
+  }
+  if (!node->is_running || !node->is_valid || ri->is_hibernating) {
+    return 0;
+  }
+  /* Only require bandwidth capacity in non-test networks, or
+   * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */
+  if (!ri->bandwidthcapacity) {
+    if (get_options()->TestingTorNetwork) {
+      if (get_options()->TestingMinExitFlagThreshold > 0) {
+        /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is,
+         * then require bandwidthcapacity */
+        return 0;
+      }
+    } else {
+      /* If we're not in a TestingTorNetwork, then require bandwidthcapacity */
+      return 0;
+    }
+  }
+  return 1;
+}
+
+/********************************************************************/
+
+/* A set of functions to answer questions about how we'd like to behave
+ * as a directory mirror/client. */
+
+/** Return 1 if we fetch our directory material directly from the
+ * authorities, rather than from a mirror. */
+int
+directory_fetches_from_authorities(const or_options_t *options)
+{
+  const routerinfo_t *me;
+  uint32_t addr;
+  int refuseunknown;
+  if (options->FetchDirInfoEarly)
+    return 1;
+  if (options->BridgeRelay == 1)
+    return 0;
+  if (server_mode(options) &&
+      router_pick_published_address(options, &addr, 1) < 0)
+    return 1; /* we don't know our IP address; ask an authority. */
+  refuseunknown = ! router_my_exit_policy_is_reject_star() &&
+    should_refuse_unknown_exits(options);
+  if (!dir_server_mode(options) && !refuseunknown)
+    return 0;
+  if (!server_mode(options) || !advertised_server_mode())
+    return 0;
+  me = router_get_my_routerinfo();
+  if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
+    return 0; /* if we don't service directory requests, return 0 too */
+  return 1;
+}
+
+/** Return 1 if we should fetch new networkstatuses, descriptors, etc
+ * on the "mirror" schedule rather than the "client" schedule.
+ */
+int
+directory_fetches_dir_info_early(const or_options_t *options)
+{
+  return directory_fetches_from_authorities(options);
+}
+
+/** Return 1 if we should fetch new networkstatuses, descriptors, etc
+ * on a very passive schedule -- waiting long enough for ordinary clients
+ * to probably have the info we want. These would include bridge users,
+ * and maybe others in the future e.g. if a Tor client uses another Tor
+ * client as a directory guard.
+ */
+int
+directory_fetches_dir_info_later(const or_options_t *options)
+{
+  return options->UseBridges != 0;
+}
+
+/** Return true iff we want to serve certificates for authorities
+ * that we don't acknowledge as authorities ourself.
+ * Use we_want_to_fetch_unknown_auth_certs to check if we want to fetch
+ * and keep these certificates.
+ */
+int
+directory_caches_unknown_auth_certs(const or_options_t *options)
+{
+  return dir_server_mode(options) || options->BridgeRelay;
+}
+
+/** Return 1 if we want to fetch and serve descriptors, networkstatuses, etc
+ * Else return 0.
+ * Check options->DirPort_set and directory_permits_begindir_requests()
+ * to see if we are willing to serve these directory documents to others via
+ * the DirPort and begindir-over-ORPort, respectively.
+ *
+ * To check if we should fetch documents, use we_want_to_fetch_flavor and
+ * we_want_to_fetch_unknown_auth_certs instead of this function.
+ */
+int
+directory_caches_dir_info(const or_options_t *options)
+{
+  if (options->BridgeRelay || dir_server_mode(options))
+    return 1;
+  if (!server_mode(options) || !advertised_server_mode())
+    return 0;
+  /* We need an up-to-date view of network info if we're going to try to
+   * block exit attempts from unknown relays. */
+  return ! router_my_exit_policy_is_reject_star() &&
+    should_refuse_unknown_exits(options);
+}
+
+/** Return 1 if we want to allow remote clients to ask us directory
+ * requests via the "begin_dir" interface, which doesn't require
+ * having any separate port open. */
+int
+directory_permits_begindir_requests(const or_options_t *options)
+{
+  return options->BridgeRelay != 0 || dir_server_mode(options);
+}
+
+/** Return 1 if we have no need to fetch new descriptors. This generally
+ * happens when we're not a dir cache and we haven't built any circuits
+ * lately.
+ */
+int
+directory_too_idle_to_fetch_descriptors(const or_options_t *options,
+                                        time_t now)
+{
+  return !directory_caches_dir_info(options) &&
+         !options->FetchUselessDescriptors &&
+         rep_hist_circbuilding_dormant(now);
+}
+
+/********************************************************************/
+
+/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
+ * currently serving. */
+static strmap_t *cached_consensuses = NULL;
+
+/** Decrement the reference count on <b>d</b>, and free it if it no longer has
+ * any references. */
+void
+cached_dir_decref(cached_dir_t *d)
+{
+  if (!d || --d->refcnt > 0)
+    return;
+  clear_cached_dir(d);
+  tor_free(d);
+}
+
+/** Allocate and return a new cached_dir_t containing the string <b>s</b>,
+ * published at <b>published</b>. */
+cached_dir_t *
+new_cached_dir(char *s, time_t published)
+{
+  cached_dir_t *d = tor_malloc_zero(sizeof(cached_dir_t));
+  d->refcnt = 1;
+  d->dir = s;
+  d->dir_len = strlen(s);
+  d->published = published;
+  if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len),
+                   d->dir, d->dir_len, ZLIB_METHOD)) {
+    log_warn(LD_BUG, "Error compressing directory");
+  }
+  return d;
+}
+
+/** Remove all storage held in <b>d</b>, but do not free <b>d</b> itself. */
+static void
+clear_cached_dir(cached_dir_t *d)
+{
+  tor_free(d->dir);
+  tor_free(d->dir_compressed);
+  memset(d, 0, sizeof(cached_dir_t));
+}
+
+/** Free all storage held by the cached_dir_t in <b>d</b>. */
+static void
+free_cached_dir_(void *_d)
+{
+  cached_dir_t *d;
+  if (!_d)
+    return;
+
+  d = (cached_dir_t *)_d;
+  cached_dir_decref(d);
+}
+
+/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that
+ * we're serving with <b>networkstatus</b>, published at <b>published</b>.  No
+ * validation is performed. */
+void
+dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
+                                           const char *flavor_name,
+                                           const common_digests_t *digests,
+                                           const uint8_t *sha3_as_signed,
+                                           time_t published)
+{
+  cached_dir_t *new_networkstatus;
+  cached_dir_t *old_networkstatus;
+  if (!cached_consensuses)
+    cached_consensuses = strmap_new();
+
+  new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
+  memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
+  memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
+         DIGEST256_LEN);
+  old_networkstatus = strmap_set(cached_consensuses, flavor_name,
+                                 new_networkstatus);
+  if (old_networkstatus)
+    cached_dir_decref(old_networkstatus);
+}
+
+/** Return the latest downloaded consensus networkstatus in encoded, signed,
+ * optionally compressed format, suitable for sending to clients. */
+cached_dir_t *
+dirserv_get_consensus(const char *flavor_name)
+{
+  if (!cached_consensuses)
+    return NULL;
+  return strmap_get(cached_consensuses, flavor_name);
+}
+
+/** If a router's uptime is at least this value, then it is always
+ * considered stable, regardless of the rest of the network. This
+ * way we resist attacks where an attacker doubles the size of the
+ * network using allegedly high-uptime nodes, displacing all the
+ * current guards. */
+#define UPTIME_TO_GUARANTEE_STABLE (3600*24*30)
+/** If a router's MTBF is at least this value, then it is always stable.
+ * See above.  (Corresponds to about 7 days for current decay rates.) */
+#define MTBF_TO_GUARANTEE_STABLE (60*60*24*5)
+/** Similarly, every node with at least this much weighted time known can be
+ * considered familiar enough to be a guard.  Corresponds to about 20 days for
+ * current decay rates.
+ */
+#define TIME_KNOWN_TO_GUARANTEE_FAMILIAR (8*24*60*60)
+/** Similarly, every node with sufficient WFU is around enough to be a guard.
+ */
+#define WFU_TO_GUARANTEE_GUARD (0.98)
+
+/* Thresholds for server performance: set by
+ * dirserv_compute_performance_thresholds, and used by
+ * generate_v2_networkstatus */
+
+/** Any router with an uptime of at least this value is stable. */
+static uint32_t stable_uptime = 0; /* start at a safe value */
+/** Any router with an mtbf of at least this value is stable. */
+static double stable_mtbf = 0.0;
+/** If true, we have measured enough mtbf info to look at stable_mtbf rather
+ * than stable_uptime. */
+static int enough_mtbf_info = 0;
+/** Any router with a weighted fractional uptime of at least this much might
+ * be good as a guard. */
+static double guard_wfu = 0.0;
+/** Don't call a router a guard unless we've known about it for at least this
+ * many seconds. */
+static long guard_tk = 0;
+/** Any router with a bandwidth at least this high is "Fast" */
+static uint32_t fast_bandwidth_kb = 0;
+/** If exits can be guards, then all guards must have a bandwidth this
+ * high. */
+static uint32_t guard_bandwidth_including_exits_kb = 0;
+/** If exits can't be guards, then all guards must have a bandwidth this
+ * high. */
+static uint32_t guard_bandwidth_excluding_exits_kb = 0;
+
+/** Helper: estimate the uptime of a router given its stated uptime and the
+ * amount of time since it last stated its stated uptime. */
+static inline long
+real_uptime(const routerinfo_t *router, time_t now)
+{
+  if (now < router->cache_info.published_on)
+    return router->uptime;
+  else
+    return router->uptime + (now - router->cache_info.published_on);
+}
+
+/** Return 1 if <b>router</b> is not suitable for these parameters, else 0.
+ * If <b>need_uptime</b> is non-zero, we require a minimum uptime.
+ * If <b>need_capacity</b> is non-zero, we require a minimum advertised
+ * bandwidth.
+ */
+static int
+dirserv_thinks_router_is_unreliable(time_t now,
+                                    routerinfo_t *router,
+                                    int need_uptime, int need_capacity)
+{
+  if (need_uptime) {
+    if (!enough_mtbf_info) {
+      /* XXXX We should change the rule from
+       * "use uptime if we don't have mtbf data" to "don't advertise Stable on
+       * v3 if we don't have enough mtbf data."  Or maybe not, since if we ever
+       * hit a point where we need to reset a lot of authorities at once,
+       * none of them would be in a position to declare Stable.
+       */
+      long uptime = real_uptime(router, now);
+      if ((unsigned)uptime < stable_uptime &&
+          (unsigned)uptime < UPTIME_TO_GUARANTEE_STABLE)
+        return 1;
+    } else {
+      double mtbf =
+        rep_hist_get_stability(router->cache_info.identity_digest, now);
+      if (mtbf < stable_mtbf &&
+          mtbf < MTBF_TO_GUARANTEE_STABLE)
+        return 1;
+    }
+  }
+  if (need_capacity) {
+    uint32_t bw_kb = dirserv_get_credible_bandwidth_kb(router);
+    if (bw_kb < fast_bandwidth_kb)
+      return 1;
+  }
+  return 0;
+}
+
+/** Return true iff <b>router</b> should be assigned the "HSDir" flag.
+ *
+ * Right now this means it advertises support for it, it has a high uptime,
+ * it's a directory cache, it has the Stable and Fast flags, and it's currently
+ * considered Running.
+ *
+ * This function needs to be called after router-\>is_running has
+ * been set.
+ */
+static int
+dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
+                                const node_t *node, time_t now)
+{
+
+  long uptime;
+
+  /* If we haven't been running for at least
+   * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't
+   * have accurate data telling us a relay has been up for at least
+   * that long. We also want to allow a bit of slack: Reachability
+   * tests aren't instant. If we haven't been running long enough,
+   * trust the relay. */
+
+  if (get_uptime() >
+      get_options()->MinUptimeHidServDirectoryV2 * 1.1)
+    uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now),
+                 real_uptime(router, now));
+  else
+    uptime = real_uptime(router, now);
+
+  return (router->wants_to_be_hs_dir &&
+          router->supports_tunnelled_dir_requests &&
+          node->is_stable && node->is_fast &&
+          uptime >= get_options()->MinUptimeHidServDirectoryV2 &&
+          router_is_active(router, node, now));
+}
+
+/** Don't consider routers with less bandwidth than this when computing
+ * thresholds. */
+#define ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB 4
+
+/** Helper for dirserv_compute_performance_thresholds(): Decide whether to
+ * include a router in our calculations, and return true iff we should; the
+ * require_mbw parameter is passed in by
+ * dirserv_compute_performance_thresholds() and controls whether we ever
+ * count routers with only advertised bandwidths */
+static int
+router_counts_toward_thresholds(const node_t *node, time_t now,
+                                const digestmap_t *omit_as_sybil,
+                                int require_mbw)
+{
+  /* Have measured bw? */
+  int have_mbw =
+    dirserv_has_measured_bw(node->identity);
+  uint64_t min_bw_kb = ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB;
+  const or_options_t *options = get_options();
+
+  if (options->TestingTorNetwork) {
+    min_bw_kb = (int64_t)options->TestingMinExitFlagThreshold / 1000;
+  }
+
+  return node->ri && router_is_active(node->ri, node, now) &&
+    !digestmap_get(omit_as_sybil, node->identity) &&
+    (dirserv_get_credible_bandwidth_kb(node->ri) >= min_bw_kb) &&
+    (have_mbw || !require_mbw);
+}
+
+/** Look through the routerlist, and using the measured bandwidth cache count
+ * how many measured bandwidths we know.  This is used to decide whether we
+ * ever trust advertised bandwidths for purposes of assigning flags. */
+void
+dirserv_count_measured_bws(const smartlist_t *routers)
+{
+  /* Initialize this first */
+  routers_with_measured_bw = 0;
+
+  /* Iterate over the routerlist and count measured bandwidths */
+  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
+    /* Check if we know a measured bandwidth for this one */
+    if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) {
+      ++routers_with_measured_bw;
+    }
+  } SMARTLIST_FOREACH_END(ri);
+}
+
+/** Look through the routerlist, the Mean Time Between Failure history, and
+ * the Weighted Fractional Uptime history, and use them to set thresholds for
+ * the Stable, Fast, and Guard flags.  Update the fields stable_uptime,
+ * stable_mtbf, enough_mtbf_info, guard_wfu, guard_tk, fast_bandwidth,
+ * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits.
+ *
+ * Also, set the is_exit flag of each router appropriately. */
+void
+dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
+{
+  int n_active, n_active_nonexit, n_familiar;
+  uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb;
+  long *tks;
+  double *mtbfs, *wfus;
+  smartlist_t *nodelist;
+  time_t now = time(NULL);
+  const or_options_t *options = get_options();
+
+  /* Require mbw? */
+  int require_mbw =
+    (routers_with_measured_bw >
+     options->MinMeasuredBWsForAuthToIgnoreAdvertised) ? 1 : 0;
+
+  /* initialize these all here, in case there are no routers */
+  stable_uptime = 0;
+  stable_mtbf = 0;
+  fast_bandwidth_kb = 0;
+  guard_bandwidth_including_exits_kb = 0;
+  guard_bandwidth_excluding_exits_kb = 0;
+  guard_tk = 0;
+  guard_wfu = 0;
+
+  nodelist_assert_ok();
+  nodelist = nodelist_get_list();
+
+  /* Initialize arrays that will hold values for each router.  We'll
+   * sort them and use that to compute thresholds. */
+  n_active = n_active_nonexit = 0;
+  /* Uptime for every active router. */
+  uptimes = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
+  /* Bandwidth for every active router. */
+  bandwidths_kb = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
+  /* Bandwidth for every active non-exit router. */
+  bandwidths_excluding_exits_kb =
+    tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
+  /* Weighted mean time between failure for each active router. */
+  mtbfs = tor_calloc(smartlist_len(nodelist), sizeof(double));
+  /* Time-known for each active router. */
+  tks = tor_calloc(smartlist_len(nodelist), sizeof(long));
+  /* Weighted fractional uptime for each active router. */
+  wfus = tor_calloc(smartlist_len(nodelist), sizeof(double));
+
+  /* Now, fill in the arrays. */
+  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
+    if (options->BridgeAuthoritativeDir &&
+        node->ri &&
+        node->ri->purpose != ROUTER_PURPOSE_BRIDGE)
+      continue;
+
+    routerinfo_t *ri = node->ri;
+    if (ri) {
+      node->is_exit = (!router_exit_policy_rejects_all(ri) &&
+                       exit_policy_is_general_exit(ri->exit_policy));
+    }
+
+    if (router_counts_toward_thresholds(node, now, omit_as_sybil,
+                                        require_mbw)) {
+      const char *id = node->identity;
+      uint32_t bw_kb;
+
+      /* resolve spurious clang shallow analysis null pointer errors */
+      tor_assert(ri);
+
+      uptimes[n_active] = (uint32_t)real_uptime(ri, now);
+      mtbfs[n_active] = rep_hist_get_stability(id, now);
+      tks  [n_active] = rep_hist_get_weighted_time_known(id, now);
+      bandwidths_kb[n_active] = bw_kb = dirserv_get_credible_bandwidth_kb(ri);
+      if (!node->is_exit || node->is_bad_exit) {
+        bandwidths_excluding_exits_kb[n_active_nonexit] = bw_kb;
+        ++n_active_nonexit;
+      }
+      ++n_active;
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* Now, compute thresholds. */
+  if (n_active) {
+    /* The median uptime is stable. */
+    stable_uptime = median_uint32(uptimes, n_active);
+    /* The median mtbf is stable, if we have enough mtbf info */
+    stable_mtbf = median_double(mtbfs, n_active);
+    /* The 12.5th percentile bandwidth is fast. */
+    fast_bandwidth_kb = find_nth_uint32(bandwidths_kb, n_active, n_active/8);
+    /* (Now bandwidths is sorted.) */
+    if (fast_bandwidth_kb < RELAY_REQUIRED_MIN_BANDWIDTH/(2 * 1000))
+      fast_bandwidth_kb = bandwidths_kb[n_active/4];
+    guard_bandwidth_including_exits_kb =
+      third_quartile_uint32(bandwidths_kb, n_active);
+    guard_tk = find_nth_long(tks, n_active, n_active/8);
+  }
+
+  if (guard_tk > TIME_KNOWN_TO_GUARANTEE_FAMILIAR)
+    guard_tk = TIME_KNOWN_TO_GUARANTEE_FAMILIAR;
+
+  {
+    /* We can vote on a parameter for the minimum and maximum. */
+#define ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG 4
+    int32_t min_fast_kb, max_fast_kb, min_fast, max_fast;
+    min_fast = networkstatus_get_param(NULL, "FastFlagMinThreshold",
+      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
+      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
+      INT32_MAX);
+    if (options->TestingTorNetwork) {
+      min_fast = (int32_t)options->TestingMinFastFlagThreshold;
+    }
+    max_fast = networkstatus_get_param(NULL, "FastFlagMaxThreshold",
+                                       INT32_MAX, min_fast, INT32_MAX);
+    min_fast_kb = min_fast / 1000;
+    max_fast_kb = max_fast / 1000;
+
+    if (fast_bandwidth_kb < (uint32_t)min_fast_kb)
+      fast_bandwidth_kb = min_fast_kb;
+    if (fast_bandwidth_kb > (uint32_t)max_fast_kb)
+      fast_bandwidth_kb = max_fast_kb;
+  }
+  /* Protect sufficiently fast nodes from being pushed out of the set
+   * of Fast nodes. */
+  if (options->AuthDirFastGuarantee &&
+      fast_bandwidth_kb > options->AuthDirFastGuarantee/1000)
+    fast_bandwidth_kb = (uint32_t)options->AuthDirFastGuarantee/1000;
+
+  /* Now that we have a time-known that 7/8 routers are known longer than,
+   * fill wfus with the wfu of every such "familiar" router. */
+  n_familiar = 0;
+
+  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
+      if (router_counts_toward_thresholds(node, now,
+                                          omit_as_sybil, require_mbw)) {
+        routerinfo_t *ri = node->ri;
+        const char *id = ri->cache_info.identity_digest;
+        long tk = rep_hist_get_weighted_time_known(id, now);
+        if (tk < guard_tk)
+          continue;
+        wfus[n_familiar++] = rep_hist_get_weighted_fractional_uptime(id, now);
+      }
+  } SMARTLIST_FOREACH_END(node);
+  if (n_familiar)
+    guard_wfu = median_double(wfus, n_familiar);
+  if (guard_wfu > WFU_TO_GUARANTEE_GUARD)
+    guard_wfu = WFU_TO_GUARANTEE_GUARD;
+
+  enough_mtbf_info = rep_hist_have_measured_enough_stability();
+
+  if (n_active_nonexit) {
+    guard_bandwidth_excluding_exits_kb =
+      find_nth_uint32(bandwidths_excluding_exits_kb,
+                      n_active_nonexit, n_active_nonexit*3/4);
+  }
+
+  log_info(LD_DIRSERV,
+      "Cutoffs: For Stable, %lu sec uptime, %lu sec MTBF. "
+      "For Fast: %lu kilobytes/sec. "
+      "For Guard: WFU %.03f%%, time-known %lu sec, "
+      "and bandwidth %lu or %lu kilobytes/sec. "
+      "We%s have enough stability data.",
+      (unsigned long)stable_uptime,
+      (unsigned long)stable_mtbf,
+      (unsigned long)fast_bandwidth_kb,
+      guard_wfu*100,
+      (unsigned long)guard_tk,
+      (unsigned long)guard_bandwidth_including_exits_kb,
+      (unsigned long)guard_bandwidth_excluding_exits_kb,
+      enough_mtbf_info ? "" : " don't");
+
+  tor_free(uptimes);
+  tor_free(mtbfs);
+  tor_free(bandwidths_kb);
+  tor_free(bandwidths_excluding_exits_kb);
+  tor_free(tks);
+  tor_free(wfus);
+}
+
+/* Use dirserv_compute_performance_thresholds() to compute the thresholds
+ * for the status flags, specifically for bridges.
+ *
+ * This is only called by a Bridge Authority from
+ * networkstatus_getinfo_by_purpose().
+ */
+void
+dirserv_compute_bridge_flag_thresholds(void)
+{
+  digestmap_t *omit_as_sybil = digestmap_new();
+  dirserv_compute_performance_thresholds(omit_as_sybil);
+  digestmap_free(omit_as_sybil, NULL);
+}
+
+/** Measured bandwidth cache entry */
+typedef struct mbw_cache_entry_s {
+  long mbw_kb;
+  time_t as_of;
+} mbw_cache_entry_t;
+
+/** Measured bandwidth cache - keys are identity_digests, values are
+ * mbw_cache_entry_t *. */
+static digestmap_t *mbw_cache = NULL;
+
+/** Store a measured bandwidth cache entry when reading the measured
+ * bandwidths file. */
+STATIC void
+dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
+                          time_t as_of)
+{
+  mbw_cache_entry_t *e = NULL;
+
+  tor_assert(parsed_line);
+
+  /* Allocate a cache if we need */
+  if (!mbw_cache) mbw_cache = digestmap_new();
+
+  /* Check if we have an existing entry */
+  e = digestmap_get(mbw_cache, parsed_line->node_id);
+  /* If we do, we can re-use it */
+  if (e) {
+    /* Check that we really are newer, and update */
+    if (as_of > e->as_of) {
+      e->mbw_kb = parsed_line->bw_kb;
+      e->as_of = as_of;
+    }
+  } else {
+    /* We'll have to insert a new entry */
+    e = tor_malloc(sizeof(*e));
+    e->mbw_kb = parsed_line->bw_kb;
+    e->as_of = as_of;
+    digestmap_set(mbw_cache, parsed_line->node_id, e);
+  }
+}
+
+/** Clear and free the measured bandwidth cache */
+void
+dirserv_clear_measured_bw_cache(void)
+{
+  if (mbw_cache) {
+    /* Free the map and all entries */
+    digestmap_free(mbw_cache, tor_free_);
+    mbw_cache = NULL;
+  }
+}
+
+/** Scan the measured bandwidth cache and remove expired entries */
+STATIC void
+dirserv_expire_measured_bw_cache(time_t now)
+{
+
+  if (mbw_cache) {
+    /* Iterate through the cache and check each entry */
+    DIGESTMAP_FOREACH_MODIFY(mbw_cache, k, mbw_cache_entry_t *, e) {
+      if (now > e->as_of + MAX_MEASUREMENT_AGE) {
+        tor_free(e);
+        MAP_DEL_CURRENT(k);
+      }
+    } DIGESTMAP_FOREACH_END;
+
+    /* Check if we cleared the whole thing and free if so */
+    if (digestmap_size(mbw_cache) == 0) {
+      digestmap_free(mbw_cache, tor_free_);
+      mbw_cache = 0;
+    }
+  }
+}
+
+/** Query the cache by identity digest, return value indicates whether
+ * we found it. The bw_out and as_of_out pointers receive the cached
+ * bandwidth value and the time it was cached if not NULL. */
+int
+dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out,
+                                   time_t *as_of_out)
+{
+  mbw_cache_entry_t *v = NULL;
+  int rv = 0;
+
+  if (mbw_cache && node_id) {
+    v = digestmap_get(mbw_cache, node_id);
+    if (v) {
+      /* Found something */
+      rv = 1;
+      if (bw_kb_out) *bw_kb_out = v->mbw_kb;
+      if (as_of_out) *as_of_out = v->as_of;
+    }
+  }
+
+  return rv;
+}
+
+/** Predicate wrapper for dirserv_query_measured_bw_cache() */
+int
+dirserv_has_measured_bw(const char *node_id)
+{
+  return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL);
+}
+
+/** Get the current size of the measured bandwidth cache */
+int
+dirserv_get_measured_bw_cache_size(void)
+{
+  if (mbw_cache) return digestmap_size(mbw_cache);
+  else return 0;
+}
+
+/** Return the bandwidth we believe for assigning flags; prefer measured
+ * over advertised, and if we have above a threshold quantity of measured
+ * bandwidths, we don't want to ever give flags to unmeasured routers, so
+ * return 0. */
+static uint32_t
+dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
+{
+  int threshold;
+  uint32_t bw_kb = 0;
+  long mbw_kb;
+
+  tor_assert(ri);
+  /* Check if we have a measured bandwidth, and check the threshold if not */
+  if (!(dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
+                                       &mbw_kb, NULL))) {
+    threshold = get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+    if (routers_with_measured_bw > threshold) {
+      /* Return zero for unmeasured bandwidth if we are above threshold */
+      bw_kb = 0;
+    } else {
+      /* Return an advertised bandwidth otherwise */
+      bw_kb = router_get_advertised_bandwidth_capped(ri) / 1000;
+    }
+  } else {
+    /* We have the measured bandwidth in mbw */
+    bw_kb = (uint32_t)mbw_kb;
+  }
+
+  return bw_kb;
+}
+
+/** Give a statement of our current performance thresholds for inclusion
+ * in a vote document. */
+char *
+dirserv_get_flag_thresholds_line(void)
+{
+  char *result=NULL;
+  const int measured_threshold =
+    get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+  const int enough_measured_bw = routers_with_measured_bw > measured_threshold;
+
+  tor_asprintf(&result,
+      "stable-uptime=%lu stable-mtbf=%lu "
+      "fast-speed=%lu "
+      "guard-wfu=%.03f%% guard-tk=%lu "
+      "guard-bw-inc-exits=%lu guard-bw-exc-exits=%lu "
+      "enough-mtbf=%d ignoring-advertised-bws=%d",
+      (unsigned long)stable_uptime,
+      (unsigned long)stable_mtbf,
+      (unsigned long)fast_bandwidth_kb*1000,
+      guard_wfu*100,
+      (unsigned long)guard_tk,
+      (unsigned long)guard_bandwidth_including_exits_kb*1000,
+      (unsigned long)guard_bandwidth_excluding_exits_kb*1000,
+      enough_mtbf_info ? 1 : 0,
+      enough_measured_bw ? 1 : 0);
+
+  return result;
+}
+
+/** Helper: write the router-status information in <b>rs</b> into a newly
+ * allocated character buffer.  Use the same format as in network-status
+ * documents.  If <b>version</b> is non-NULL, add a "v" line for the platform.
+ *
+ * consensus_method is the current consensus method when format is
+ * NS_V3_CONSENSUS or NS_V3_CONSENSUS_MICRODESC. It is ignored for other
+ * formats: pass ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD.
+ *
+ * Return 0 on success, -1 on failure.
+ *
+ * The format argument has one of the following values:
+ *   NS_V2 - Output an entry suitable for a V2 NS opinion document
+ *   NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry
+ *        for consensus_method.
+ *   NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc
+ *        consensus entry for consensus_method.
+ *   NS_V3_VOTE - Output a complete V3 NS vote. If <b>vrs</b> is present,
+ *        it contains additional information for the vote.
+ *   NS_CONTROL_PORT - Output a NS document for the control port.
+ */
+char *
+routerstatus_format_entry(const routerstatus_t *rs, const char *version,
+                          const char *protocols,
+                          routerstatus_format_type_t format,
+                          int consensus_method,
+                          const vote_routerstatus_t *vrs)
+{
+  char *summary;
+  char *result = NULL;
+
+  char published[ISO_TIME_LEN+1];
+  char identity64[BASE64_DIGEST_LEN+1];
+  char digest64[BASE64_DIGEST_LEN+1];
+  smartlist_t *chunks = smartlist_new();
+
+  format_iso_time(published, rs->published_on);
+  digest_to_base64(identity64, rs->identity_digest);
+  digest_to_base64(digest64, rs->descriptor_digest);
+
+  smartlist_add_asprintf(chunks,
+                   "r %s %s %s%s%s %s %d %d\n",
+                   rs->nickname,
+                   identity64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
+                   published,
+                   fmt_addr32(rs->addr),
+                   (int)rs->or_port,
+                   (int)rs->dir_port);
+
+  /* TODO: Maybe we want to pass in what we need to build the rest of
+   * this here, instead of in the caller. Then we could use the
+   * networkstatus_type_t values, with an additional control port value
+   * added -MP */
+
+  /* V3 microdesc consensuses only have "a" lines in later consensus methods
+   */
+  if (format == NS_V3_CONSENSUS_MICRODESC &&
+      consensus_method < MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS)
+    goto done;
+
+  /* Possible "a" line. At most one for now. */
+  if (!tor_addr_is_null(&rs->ipv6_addr)) {
+    smartlist_add_asprintf(chunks, "a %s\n",
+                           fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport));
+  }
+
+  if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC)
+    goto done;
+
+  smartlist_add_asprintf(chunks,
+                   "s%s%s%s%s%s%s%s%s%s%s\n",
+                  /* These must stay in alphabetical order. */
+                   rs->is_authority?" Authority":"",
+                   rs->is_bad_exit?" BadExit":"",
+                   rs->is_exit?" Exit":"",
+                   rs->is_fast?" Fast":"",
+                   rs->is_possible_guard?" Guard":"",
+                   rs->is_hs_dir?" HSDir":"",
+                   rs->is_flagged_running?" Running":"",
+                   rs->is_stable?" Stable":"",
+                   rs->is_v2_dir?" V2Dir":"",
+                   rs->is_valid?" Valid":"");
+
+  /* length of "opt v \n" */
+#define V_LINE_OVERHEAD 7
+  if (version && strlen(version) < MAX_V_LINE_LEN - V_LINE_OVERHEAD) {
+    smartlist_add_asprintf(chunks, "v %s\n", version);
+  }
+  if (protocols) {
+    smartlist_add_asprintf(chunks, "pr %s\n", protocols);
+  }
+
+  if (format != NS_V2) {
+    const routerinfo_t* desc = router_get_by_id_digest(rs->identity_digest);
+    uint32_t bw_kb;
+
+    if (format != NS_CONTROL_PORT) {
+      /* Blow up more or less nicely if we didn't get anything or not the
+       * thing we expected.
+       */
+      if (!desc) {
+        char id[HEX_DIGEST_LEN+1];
+        char dd[HEX_DIGEST_LEN+1];
+
+        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
+        base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN);
+        log_warn(LD_BUG, "Cannot get any descriptor for %s "
+            "(wanted descriptor %s).",
+            id, dd);
+        goto err;
+      }
+
+      /* This assert could fire for the control port, because
+       * it can request NS documents before all descriptors
+       * have been fetched. Therefore, we only do this test when
+       * format != NS_CONTROL_PORT. */
+      if (tor_memneq(desc->cache_info.signed_descriptor_digest,
+            rs->descriptor_digest,
+            DIGEST_LEN)) {
+        char rl_d[HEX_DIGEST_LEN+1];
+        char rs_d[HEX_DIGEST_LEN+1];
+        char id[HEX_DIGEST_LEN+1];
+
+        base16_encode(rl_d, sizeof(rl_d),
+            desc->cache_info.signed_descriptor_digest, DIGEST_LEN);
+        base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN);
+        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
+        log_err(LD_BUG, "descriptor digest in routerlist does not match "
+            "the one in routerstatus: %s vs %s "
+            "(router %s)\n",
+            rl_d, rs_d, id);
+
+        tor_assert(tor_memeq(desc->cache_info.signed_descriptor_digest,
+              rs->descriptor_digest,
+              DIGEST_LEN));
+      }
+    }
+
+    if (format == NS_CONTROL_PORT && rs->has_bandwidth) {
+      bw_kb = rs->bandwidth_kb;
+    } else {
+      tor_assert(desc);
+      bw_kb = router_get_advertised_bandwidth_capped(desc) / 1000;
+    }
+    smartlist_add_asprintf(chunks,
+                     "w Bandwidth=%d", bw_kb);
+
+    if (format == NS_V3_VOTE && vrs && vrs->has_measured_bw) {
+      smartlist_add_asprintf(chunks,
+                       " Measured=%d", vrs->measured_bw_kb);
+    }
+    /* Write down guardfraction information if we have it. */
+    if (format == NS_V3_VOTE && vrs && vrs->status.has_guardfraction) {
+      smartlist_add_asprintf(chunks,
+                             " GuardFraction=%d",
+                             vrs->status.guardfraction_percentage);
+    }
+
+    smartlist_add_strdup(chunks, "\n");
+
+    if (desc) {
+      summary = policy_summarize(desc->exit_policy, AF_INET);
+      smartlist_add_asprintf(chunks, "p %s\n", summary);
+      tor_free(summary);
+    }
+
+    if (format == NS_V3_VOTE && vrs) {
+      if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
+        smartlist_add_strdup(chunks, "id ed25519 none\n");
+      } else {
+        char ed_b64[BASE64_DIGEST256_LEN+1];
+        digest256_to_base64(ed_b64, (const char*)vrs->ed25519_id);
+        smartlist_add_asprintf(chunks, "id ed25519 %s\n", ed_b64);
+      }
+    }
+  }
+
+ done:
+  result = smartlist_join_strings(chunks, "", 0, NULL);
+
+ err:
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+
+  return result;
+}
+
+/** Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
+ * set.
+ *
+ * We assume that ri-\>is_running has already been set, e.g. by
+ *   dirserv_set_router_is_running(ri, now);
+ */
+void
+set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                 node_t *node,
+                                 routerinfo_t *ri,
+                                 time_t now,
+                                 int listbadexits)
+{
+  const or_options_t *options = get_options();
+  uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri);
+
+  memset(rs, 0, sizeof(routerstatus_t));
+
+  rs->is_authority =
+    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
+
+  /* Already set by compute_performance_thresholds. */
+  rs->is_exit = node->is_exit;
+  rs->is_stable = node->is_stable =
+    !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
+  rs->is_fast = node->is_fast =
+    !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
+  rs->is_flagged_running = node->is_running; /* computed above */
+
+  rs->is_valid = node->is_valid;
+
+  if (node->is_fast && node->is_stable &&
+      ri->supports_tunnelled_dir_requests &&
+      ((options->AuthDirGuardBWGuarantee &&
+        routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) ||
+       routerbw_kb >= MIN(guard_bandwidth_including_exits_kb,
+                          guard_bandwidth_excluding_exits_kb))) {
+    long tk = rep_hist_get_weighted_time_known(
+                                      node->identity, now);
+    double wfu = rep_hist_get_weighted_fractional_uptime(
+                                      node->identity, now);
+    rs->is_possible_guard = (wfu >= guard_wfu && tk >= guard_tk) ? 1 : 0;
+  } else {
+    rs->is_possible_guard = 0;
+  }
+
+  rs->is_bad_exit = listbadexits && node->is_bad_exit;
+  rs->is_hs_dir = node->is_hs_dir =
+    dirserv_thinks_router_is_hs_dir(ri, node, now);
+
+  rs->is_named = rs->is_unnamed = 0;
+
+  rs->published_on = ri->cache_info.published_on;
+  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
+  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  rs->addr = ri->addr;
+  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
+  rs->or_port = ri->or_port;
+  rs->dir_port = ri->dir_port;
+  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
+  if (options->AuthDirHasIPv6Connectivity == 1 &&
+      !tor_addr_is_null(&ri->ipv6_addr) &&
+      node->last_reachable6 >= now - REACHABLE_TIMEOUT) {
+    /* We're configured as having IPv6 connectivity. There's an IPv6
+       OR port and it's reachable so copy it to the routerstatus.  */
+    tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
+    rs->ipv6_orport = ri->ipv6_orport;
+  } else {
+    tor_addr_make_null(&rs->ipv6_addr, AF_INET6);
+    rs->ipv6_orport = 0;
+  }
+
+  if (options->TestingTorNetwork) {
+    dirserv_set_routerstatus_testing(rs);
+  }
+}
+
+/** Use TestingDirAuthVoteExit, TestingDirAuthVoteGuard, and
+ * TestingDirAuthVoteHSDir to give out the Exit, Guard, and HSDir flags,
+ * respectively. But don't set the corresponding node flags.
+ * Should only be called if TestingTorNetwork is set. */
+STATIC void
+dirserv_set_routerstatus_testing(routerstatus_t *rs)
+{
+  const or_options_t *options = get_options();
+
+  tor_assert(options->TestingTorNetwork);
+
+  if (routerset_contains_routerstatus(options->TestingDirAuthVoteExit,
+                                      rs, 0)) {
+    rs->is_exit = 1;
+  } else if (options->TestingDirAuthVoteExitIsStrict) {
+    rs->is_exit = 0;
+  }
+
+  if (routerset_contains_routerstatus(options->TestingDirAuthVoteGuard,
+                                      rs, 0)) {
+    rs->is_possible_guard = 1;
+  } else if (options->TestingDirAuthVoteGuardIsStrict) {
+    rs->is_possible_guard = 0;
+  }
+
+  if (routerset_contains_routerstatus(options->TestingDirAuthVoteHSDir,
+                                      rs, 0)) {
+    rs->is_hs_dir = 1;
+  } else if (options->TestingDirAuthVoteHSDirIsStrict) {
+    rs->is_hs_dir = 0;
+  }
+}
+
+/** The guardfraction of the guard with identity fingerprint <b>guard_id</b>
+ *  is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for
+ *  this guard in <b>vote_routerstatuses</b>, and if we do, register the
+ *  information to it.
+ *
+ *  Return 1 if we applied the information and 0 if we couldn't find a
+ *  matching guard.
+ *
+ * Requires that <b>vote_routerstatuses</b> be sorted.
+ */
+static int
+guardfraction_line_apply(const char *guard_id,
+                      uint32_t guardfraction_percentage,
+                      smartlist_t *vote_routerstatuses)
+{
+  vote_routerstatus_t *vrs = NULL;
+
+  tor_assert(vote_routerstatuses);
+
+  vrs = smartlist_bsearch(vote_routerstatuses, guard_id,
+                         compare_digest_to_vote_routerstatus_entry);
+
+  if (!vrs) {
+    return 0;
+  }
+
+  vrs->status.has_guardfraction = 1;
+  vrs->status.guardfraction_percentage = guardfraction_percentage;
+
+  return 1;
+}
+
+/* Given a guard line from a guardfraction file, parse it and register
+ * its information to <b>vote_routerstatuses</b>.
+ *
+ * Return:
+ * * 1 if the line was proper and its information got registered.
+ * * 0 if the line was proper but no currently active guard was found
+ *     to register the guardfraction information to.
+ * * -1 if the line could not be parsed and set <b>err_msg</b> to a
+      newly allocated string containing the error message.
+ */
+static int
+guardfraction_file_parse_guard_line(const char *guard_line,
+                                    smartlist_t *vote_routerstatuses,
+                                    char **err_msg)
+{
+  char guard_id[DIGEST_LEN];
+  uint32_t guardfraction;
+  char *inputs_tmp = NULL;
+  int num_ok = 1;
+
+  smartlist_t *sl = smartlist_new();
+  int retval = -1;
+
+  tor_assert(err_msg);
+
+  /* guard_line should contain something like this:
+     <hex digest> <guardfraction> <appearances> */
+  smartlist_split_string(sl, guard_line, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+  if (smartlist_len(sl) < 3) {
+    tor_asprintf(err_msg, "bad line '%s'", guard_line);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 0);
+  if (strlen(inputs_tmp) != HEX_DIGEST_LEN ||
+      base16_decode(guard_id, DIGEST_LEN,
+                    inputs_tmp, HEX_DIGEST_LEN) != DIGEST_LEN) {
+    tor_asprintf(err_msg, "bad digest '%s'", inputs_tmp);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 1);
+  /* Guardfraction is an integer in [0, 100]. */
+  guardfraction =
+    (uint32_t) tor_parse_long(inputs_tmp, 10, 0, 100, &num_ok, NULL);
+  if (!num_ok) {
+    tor_asprintf(err_msg, "wrong percentage '%s'", inputs_tmp);
+    goto done;
+  }
+
+  /* If routerstatuses were provided, apply this info to actual routers. */
+  if (vote_routerstatuses) {
+    retval = guardfraction_line_apply(guard_id, guardfraction,
+                                      vote_routerstatuses);
+  } else {
+    retval = 0; /* If we got this far, line was correctly formatted. */
+  }
+
+ done:
+
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+
+  return retval;
+}
+
+/** Given an inputs line from a guardfraction file, parse it and
+ *  register its information to <b>total_consensuses</b> and
+ *  <b>total_days</b>.
+ *
+ *  Return 0 if it parsed well. Return -1 if there was an error, and
+ *  set <b>err_msg</b> to a newly allocated string containing the
+ *  error message.
+ */
+static int
+guardfraction_file_parse_inputs_line(const char *inputs_line,
+                                     int *total_consensuses,
+                                     int *total_days,
+                                     char **err_msg)
+{
+  int retval = -1;
+  char *inputs_tmp = NULL;
+  int num_ok = 1;
+  smartlist_t *sl = smartlist_new();
+
+  tor_assert(err_msg);
+
+  /* Second line is inputs information:
+   *   n-inputs <total_consensuses> <total_days>. */
+  smartlist_split_string(sl, inputs_line, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+  if (smartlist_len(sl) < 2) {
+    tor_asprintf(err_msg, "incomplete line '%s'", inputs_line);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 0);
+  *total_consensuses =
+    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
+  if (!num_ok) {
+    tor_asprintf(err_msg, "unparseable consensus '%s'", inputs_tmp);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 1);
+  *total_days =
+    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
+  if (!num_ok) {
+    tor_asprintf(err_msg, "unparseable days '%s'", inputs_tmp);
+    goto done;
+  }
+
+  retval = 0;
+
+ done:
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+
+  return retval;
+}
+
+/* Maximum age of a guardfraction file that we are willing to accept. */
+#define MAX_GUARDFRACTION_FILE_AGE (7*24*60*60) /* approx a week */
+
+/** Static strings of guardfraction files. */
+#define GUARDFRACTION_DATE_STR "written-at"
+#define GUARDFRACTION_INPUTS "n-inputs"
+#define GUARDFRACTION_GUARD "guard-seen"
+#define GUARDFRACTION_VERSION "guardfraction-file-version"
+
+/** Given a guardfraction file in a string, parse it and register the
+ *  guardfraction information to the provided vote routerstatuses.
+ *
+ *  This is the rough format of the guardfraction file:
+ *
+ *      guardfraction-file-version 1
+ *      written-at <date and time>
+ *      n-inputs <number of consesuses parsed> <number of days considered>
+ *
+ *      guard-seen <fpr 1> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 2> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 3> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 4> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 5> <guardfraction percentage> <consensus appearances>
+ *      ...
+ *
+ *  Return -1 if the parsing failed and 0 if it went smoothly. Parsing
+ *  should tolerate errors in all lines but the written-at header.
+ */
+STATIC int
+dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
+                                      smartlist_t *vote_routerstatuses)
+{
+  config_line_t *front=NULL, *line;
+  int ret_tmp;
+  int retval = -1;
+  int current_line_n = 0; /* line counter for better log messages */
+
+  /* Guardfraction info to be parsed */
+  int total_consensuses = 0;
+  int total_days = 0;
+
+  /* Stats */
+  int guards_read_n = 0;
+  int guards_applied_n = 0;
+
+  /* Parse file and split it in lines */
+  ret_tmp = config_get_lines(guardfraction_file_str, &front, 0);
+  if (ret_tmp < 0) {
+    log_warn(LD_CONFIG, "Error reading from guardfraction file");
+    goto done;
+  }
+
+  /* Sort routerstatuses (needed later when applying guardfraction info) */
+  if (vote_routerstatuses)
+    smartlist_sort(vote_routerstatuses, compare_vote_routerstatus_entries);
+
+  for (line = front; line; line=line->next) {
+    current_line_n++;
+
+    if (!strcmp(line->key, GUARDFRACTION_VERSION)) {
+      int num_ok = 1;
+      unsigned int version;
+
+      version =
+        (unsigned int) tor_parse_long(line->value,
+                                      10, 0, INT_MAX, &num_ok, NULL);
+
+      if (!num_ok || version != 1) {
+        log_warn(LD_GENERAL, "Got unknown guardfraction version %d.", version);
+        goto done;
+      }
+    } else if (!strcmp(line->key, GUARDFRACTION_DATE_STR)) {
+      time_t file_written_at;
+      time_t now = time(NULL);
+
+      /* First line is 'written-at <date>' */
+      if (parse_iso_time(line->value, &file_written_at) < 0) {
+        log_warn(LD_CONFIG, "Guardfraction:%d: Bad date '%s'. Ignoring",
+                 current_line_n, line->value);
+        goto done; /* don't tolerate failure here. */
+      }
+      if (file_written_at < now - MAX_GUARDFRACTION_FILE_AGE) {
+        log_warn(LD_CONFIG, "Guardfraction:%d: was written very long ago '%s'",
+                 current_line_n, line->value);
+        goto done; /* don't tolerate failure here. */
+      }
+    } else if (!strcmp(line->key, GUARDFRACTION_INPUTS)) {
+      char *err_msg = NULL;
+
+      if (guardfraction_file_parse_inputs_line(line->value,
+                                               &total_consensuses,
+                                               &total_days,
+                                               &err_msg) < 0) {
+        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
+                 current_line_n, err_msg);
+        tor_free(err_msg);
+        continue;
+      }
+
+    } else if (!strcmp(line->key, GUARDFRACTION_GUARD)) {
+      char *err_msg = NULL;
+
+      ret_tmp = guardfraction_file_parse_guard_line(line->value,
+                                                    vote_routerstatuses,
+                                                    &err_msg);
+      if (ret_tmp < 0) { /* failed while parsing the guard line */
+        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
+                 current_line_n, err_msg);
+        tor_free(err_msg);
+        continue;
+      }
+
+      /* Successfully parsed guard line. Check if it was applied properly. */
+      guards_read_n++;
+      if (ret_tmp > 0) {
+        guards_applied_n++;
+      }
+    } else {
+      log_warn(LD_CONFIG, "Unknown guardfraction line %d (%s %s)",
+               current_line_n, line->key, line->value);
+    }
+  }
+
+  retval = 0;
+
+  log_info(LD_CONFIG,
+           "Successfully parsed guardfraction file with %d consensuses over "
+           "%d days. Parsed %d nodes and applied %d of them%s.",
+           total_consensuses, total_days, guards_read_n, guards_applied_n,
+           vote_routerstatuses ? "" : " (no routerstatus provided)" );
+
+ done:
+  config_free_lines(front);
+
+  if (retval < 0) {
+    return retval;
+  } else {
+    return guards_read_n;
+  }
+}
+
+/** Read a guardfraction file at <b>fname</b> and load all its
+ *  information to <b>vote_routerstatuses</b>. */
+int
+dirserv_read_guardfraction_file(const char *fname,
+                             smartlist_t *vote_routerstatuses)
+{
+  char *guardfraction_file_str;
+
+  /* Read file to a string */
+  guardfraction_file_str = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+  if (!guardfraction_file_str) {
+      log_warn(LD_FS, "Cannot open guardfraction file '%s'. Failing.", fname);
+      return -1;
+  }
+
+  return dirserv_read_guardfraction_file_from_str(guardfraction_file_str,
+                                               vote_routerstatuses);
+}
+
+/**
+ * Helper function to parse out a line in the measured bandwidth file
+ * into a measured_bw_line_t output structure.
+ *
+ * If <b>line_is_after_headers</b> is true, then if we encounter an incomplete
+ * bw line, return -1 and warn, since we are after the headers and we should
+ * only parse bw lines. Return 0 otherwise.
+ *
+ * If <b>line_is_after_headers</b> is false then it means that we are not past
+ * the header block yet. If we encounter an incomplete bw line, return -1 but
+ * don't warn since there could be additional header lines coming. If we
+ * encounter a proper bw line, return 0 (and we got past the headers).
+ */
+STATIC int
+measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
+                       int line_is_after_headers)
+{
+  char *line = tor_strdup(orig_line);
+  char *cp = line;
+  int got_bw = 0;
+  int got_node_id = 0;
+  char *strtok_state; /* lame sauce d'jour */
+
+  if (strlen(line) == 0) {
+    log_warn(LD_DIRSERV, "Empty line in bandwidth file");
+    tor_free(line);
+    return -1;
+  }
+
+  /* Remove end of line character, so that is not part of the token */
+  if (line[strlen(line) - 1] == '\n') {
+    line[strlen(line) - 1] = '\0';
+  }
+
+  cp = tor_strtok_r(cp, " \t", &strtok_state);
+
+  if (!cp) {
+    log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  }
+
+  if (orig_line[strlen(orig_line)-1] != '\n') {
+    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  }
+
+  do {
+    if (strcmpstart(cp, "bw=") == 0) {
+      int parse_ok = 0;
+      char *endptr;
+      if (got_bw) {
+        log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      cp+=strlen("bw=");
+
+      out->bw_kb = tor_parse_long(cp, 10, 0, LONG_MAX, &parse_ok, &endptr);
+      if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) {
+        log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      got_bw=1;
+    } else if (strcmpstart(cp, "node_id=$") == 0) {
+      if (got_node_id) {
+        log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      cp+=strlen("node_id=$");
+
+      if (strlen(cp) != HEX_DIGEST_LEN ||
+          base16_decode(out->node_id, DIGEST_LEN,
+                        cp, HEX_DIGEST_LEN) != DIGEST_LEN) {
+        log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      strlcpy(out->node_hex, cp, sizeof(out->node_hex));
+      got_node_id=1;
+    }
+  } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state)));
+
+  if (got_bw && got_node_id) {
+    tor_free(line);
+    return 0;
+  } else if (line_is_after_headers == 0) {
+    /* There could be additional header lines, therefore do not give warnings
+     * but returns -1 since it's not a complete bw line. */
+    log_debug(LD_DIRSERV, "Missing bw or node_id in bandwidth file line: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  } else {
+    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  }
+}
+
+/**
+ * Helper function to apply a parsed measurement line to a list
+ * of bandwidth statuses. Returns true if a line is found,
+ * false otherwise.
+ */
+STATIC int
+measured_bw_line_apply(measured_bw_line_t *parsed_line,
+                       smartlist_t *routerstatuses)
+{
+  vote_routerstatus_t *rs = NULL;
+  if (!routerstatuses)
+    return 0;
+
+  rs = smartlist_bsearch(routerstatuses, parsed_line->node_id,
+                         compare_digest_to_vote_routerstatus_entry);
+
+  if (rs) {
+    rs->has_measured_bw = 1;
+    rs->measured_bw_kb = (uint32_t)parsed_line->bw_kb;
+  } else {
+    log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list",
+             parsed_line->node_hex);
+  }
+
+  return rs != NULL;
+}
+
+/**
+ * Read the measured bandwidth file and apply it to the list of
+ * vote_routerstatus_t. Returns -1 on error, 0 otherwise.
+ */
+int
+dirserv_read_measured_bandwidths(const char *from_file,
+                                 smartlist_t *routerstatuses)
+{
+  char line[512];
+  FILE *fp = tor_fopen_cloexec(from_file, "r");
+  int applied_lines = 0;
+  time_t file_time, now;
+  int ok;
+   /* This flag will be 1 only when the first successful bw measurement line
+   * has been encountered, so that measured_bw_line_parse don't give warnings
+   * if there are additional header lines, as introduced in Bandwidth List spec
+   * version 1.1.0 */
+  int line_is_after_headers = 0;
+
+  /* Initialise line, so that we can't possibly run off the end. */
+  memset(line, 0, sizeof(line));
+
+  if (fp == NULL) {
+    log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
+             from_file);
+    return -1;
+  }
+
+  /* If fgets fails, line is either unmodified, or indeterminate. */
+  if (!fgets(line, sizeof(line), fp)) {
+    log_warn(LD_DIRSERV, "Empty bandwidth file");
+    fclose(fp);
+    return -1;
+  }
+
+  if (!strlen(line) || line[strlen(line)-1] != '\n') {
+    log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
+             escaped(line));
+    fclose(fp);
+    return -1;
+  }
+
+  line[strlen(line)-1] = '\0';
+  file_time = (time_t)tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
+             escaped(line));
+    fclose(fp);
+    return -1;
+  }
+
+  now = time(NULL);
+  if ((now - file_time) > MAX_MEASUREMENT_AGE) {
+    log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
+             (unsigned)(time(NULL) - file_time));
+    fclose(fp);
+    return -1;
+  }
+
+  if (routerstatuses)
+    smartlist_sort(routerstatuses, compare_vote_routerstatus_entries);
+
+  while (!feof(fp)) {
+    measured_bw_line_t parsed_line;
+    if (fgets(line, sizeof(line), fp) && strlen(line)) {
+      if (measured_bw_line_parse(&parsed_line, line,
+                                 line_is_after_headers) != -1) {
+        /* This condition will be true when the first complete valid bw line
+         * has been encountered, which means the end of the header lines. */
+        line_is_after_headers = 1;
+        /* Also cache the line for dirserv_get_bandwidth_for_router() */
+        dirserv_cache_measured_bw(&parsed_line, file_time);
+        if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0)
+          applied_lines++;
+      }
+    }
+  }
+
+  /* Now would be a nice time to clean the cache, too */
+  dirserv_expire_measured_bw_cache(now);
+
+  fclose(fp);
+  log_info(LD_DIRSERV,
+           "Bandwidth measurement file successfully read. "
+           "Applied %d measurements.", applied_lines);
+  return 0;
+}
+
+/** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t
+ * pointers, adds copies of digests to fps_out, and doesn't use the
+ * /tor/server/ prefix.  For a /d/ request, adds descriptor digests; for other
+ * requests, adds identity digests.
+ */
+int
+dirserv_get_routerdesc_spool(smartlist_t *spool_out,
+                             const char *key,
+                             dir_spool_source_t source,
+                             int conn_is_encrypted,
+                             const char **msg_out)
+{
+  *msg_out = NULL;
+
+  if (!strcmp(key, "all")) {
+    const routerlist_t *rl = router_get_routerlist();
+    SMARTLIST_FOREACH_BEGIN(rl->routers, const routerinfo_t *, r) {
+      spooled_resource_t *spooled;
+      spooled = spooled_resource_new(source,
+                              (const uint8_t *)r->cache_info.identity_digest,
+                              DIGEST_LEN);
+      /* Treat "all" requests as if they were unencrypted */
+      conn_is_encrypted = 0;
+      smartlist_add(spool_out, spooled);
+    } SMARTLIST_FOREACH_END(r);
+  } else if (!strcmp(key, "authority")) {
+    const routerinfo_t *ri = router_get_my_routerinfo();
+    if (ri)
+      smartlist_add(spool_out,
+                    spooled_resource_new(source,
+                             (const uint8_t *)ri->cache_info.identity_digest,
+                             DIGEST_LEN));
+  } else if (!strcmpstart(key, "d/")) {
+    key += strlen("d/");
+    dir_split_resource_into_spoolable(key, source, spool_out, NULL,
+                                  DSR_HEX|DSR_SORT_UNIQ);
+  } else if (!strcmpstart(key, "fp/")) {
+    key += strlen("fp/");
+    dir_split_resource_into_spoolable(key, source, spool_out, NULL,
+                                  DSR_HEX|DSR_SORT_UNIQ);
+  } else {
+    *msg_out = "Not found";
+    return -1;
+  }
+
+  if (! conn_is_encrypted) {
+    /* Remove anything that insists it not be sent unencrypted. */
+    SMARTLIST_FOREACH_BEGIN(spool_out, spooled_resource_t *, spooled) {
+      const uint8_t *body = NULL;
+      size_t bodylen = 0;
+      int r = spooled_resource_lookup_body(spooled, conn_is_encrypted,
+                                           &body, &bodylen, NULL);
+      if (r < 0 || body == NULL || bodylen == 0) {
+        SMARTLIST_DEL_CURRENT(spool_out, spooled);
+        spooled_resource_free(spooled);
+      }
+    } SMARTLIST_FOREACH_END(spooled);
+  }
+
+  if (!smartlist_len(spool_out)) {
+    *msg_out = "Servers unavailable";
+    return -1;
+  }
+  return 0;
+}
+
+/** Add a signed_descriptor_t to <b>descs_out</b> for each router matching
+ * <b>key</b>.  The key should be either
+ *   - "/tor/server/authority" for our own routerinfo;
+ *   - "/tor/server/all" for all the routerinfos we have, concatenated;
+ *   - "/tor/server/fp/FP" where FP is a plus-separated sequence of
+ *     hex identity digests; or
+ *   - "/tor/server/d/D" where D is a plus-separated sequence
+ *     of server descriptor digests, in hex.
+ *
+ * Return 0 if we found some matching descriptors, or -1 if we do not
+ * have any descriptors, no matching descriptors, or if we did not
+ * recognize the key (URL).
+ * If -1 is returned *<b>msg</b> will be set to an appropriate error
+ * message.
+ *
+ * XXXX rename this function.  It's only called from the controller.
+ * XXXX in fact, refactor this function, merging as much as possible.
+ */
+int
+dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
+                        const char **msg)
+{
+  *msg = NULL;
+
+  if (!strcmp(key, "/tor/server/all")) {
+    routerlist_t *rl = router_get_routerlist();
+    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
+                      smartlist_add(descs_out, &(r->cache_info)));
+  } else if (!strcmp(key, "/tor/server/authority")) {
+    const routerinfo_t *ri = router_get_my_routerinfo();
+    if (ri)
+      smartlist_add(descs_out, (void*) &(ri->cache_info));
+  } else if (!strcmpstart(key, "/tor/server/d/")) {
+    smartlist_t *digests = smartlist_new();
+    key += strlen("/tor/server/d/");
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH(digests, const char *, d,
+       {
+         signed_descriptor_t *sd = router_get_by_descriptor_digest(d);
+         if (sd)
+           smartlist_add(descs_out,sd);
+       });
+    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
+    smartlist_free(digests);
+  } else if (!strcmpstart(key, "/tor/server/fp/")) {
+    smartlist_t *digests = smartlist_new();
+    time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+    key += strlen("/tor/server/fp/");
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH_BEGIN(digests, const char *, d) {
+         if (router_digest_is_me(d)) {
+           /* calling router_get_my_routerinfo() to make sure it exists */
+           const routerinfo_t *ri = router_get_my_routerinfo();
+           if (ri)
+             smartlist_add(descs_out, (void*) &(ri->cache_info));
+         } else {
+           const routerinfo_t *ri = router_get_by_id_digest(d);
+           /* Don't actually serve a descriptor that everyone will think is
+            * expired.  This is an (ugly) workaround to keep buggy 0.1.1.10
+            * Tors from downloading descriptors that they will throw away.
+            */
+           if (ri && ri->cache_info.published_on > cutoff)
+             smartlist_add(descs_out, (void*) &(ri->cache_info));
+         }
+    } SMARTLIST_FOREACH_END(d);
+    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
+    smartlist_free(digests);
+  } else {
+    *msg = "Key not recognized";
+    return -1;
+  }
+
+  if (!smartlist_len(descs_out)) {
+    *msg = "Servers unavailable";
+    return -1;
+  }
+  return 0;
+}
+
+/** Called when a TLS handshake has completed successfully with a
+ * router listening at <b>address</b>:<b>or_port</b>, and has yielded
+ * a certificate with digest <b>digest_rcvd</b>.
+ *
+ * Inform the reachability checker that we could get to this relay.
+ */
+void
+dirserv_orconn_tls_done(const tor_addr_t *addr,
+                        uint16_t or_port,
+                        const char *digest_rcvd,
+                        const ed25519_public_key_t *ed_id_rcvd)
+{
+  node_t *node = NULL;
+  tor_addr_port_t orport;
+  routerinfo_t *ri = NULL;
+  time_t now = time(NULL);
+  tor_assert(addr);
+  tor_assert(digest_rcvd);
+
+  node = node_get_mutable_by_id(digest_rcvd);
+  if (node == NULL || node->ri == NULL)
+    return;
+
+  ri = node->ri;
+
+  if (get_options()->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node, 1) &&
+      ri->cache_info.signing_key_cert) {
+    /* We allow the node to have an ed25519 key if we haven't been told one in
+     * the routerinfo, but if we *HAVE* been told one in the routerinfo, it
+     * needs to match. */
+    const ed25519_public_key_t *expected_id =
+      &ri->cache_info.signing_key_cert->signing_key;
+    tor_assert(!ed25519_public_key_is_zero(expected_id));
+    if (! ed_id_rcvd || ! ed25519_pubkey_eq(ed_id_rcvd, expected_id)) {
+      log_info(LD_DIRSERV, "Router at %s:%d with RSA ID %s "
+               "did not present expected Ed25519 ID.",
+               fmt_addr(addr), or_port, hex_str(digest_rcvd, DIGEST_LEN));
+      return; /* Don't mark it as reachable. */
+    }
+  }
+
+  tor_addr_copy(&orport.addr, addr);
+  orport.port = or_port;
+  if (router_has_orport(ri, &orport)) {
+    /* Found the right router.  */
+    if (!authdir_mode_bridge(get_options()) ||
+        ri->purpose == ROUTER_PURPOSE_BRIDGE) {
+      char addrstr[TOR_ADDR_BUF_LEN];
+      /* This is a bridge or we're not a bridge authority --
+         mark it as reachable.  */
+      log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.",
+               router_describe(ri),
+               tor_addr_to_str(addrstr, addr, sizeof(addrstr), 1),
+               ri->or_port);
+      if (tor_addr_family(addr) == AF_INET) {
+        rep_hist_note_router_reachable(digest_rcvd, addr, or_port, now);
+        node->last_reachable = now;
+      } else if (tor_addr_family(addr) == AF_INET6) {
+        /* No rephist for IPv6.  */
+        node->last_reachable6 = now;
+      }
+    }
+  }
+}
+
+/** Called when we, as an authority, receive a new router descriptor either as
+ * an upload or a download.  Used to decide whether to relaunch reachability
+ * testing for the server. */
+int
+dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                        const routerinfo_t *ri_old)
+{
+  if (!authdir_mode_handles_descs(get_options(), ri->purpose))
+    return 0;
+  if (!ri_old) {
+    /* New router: Launch an immediate reachability test, so we will have an
+     * opinion soon in case we're generating a consensus soon */
+    return 1;
+  }
+  if (ri_old->is_hibernating && !ri->is_hibernating) {
+    /* It just came out of hibernation; launch a reachability test */
+    return 1;
+  }
+  if (! routers_have_same_or_addrs(ri, ri_old)) {
+    /* Address or port changed; launch a reachability test */
+    return 1;
+  }
+  return 0;
+}
+
+/** Helper function for dirserv_test_reachability(). Start a TLS
+ * connection to <b>router</b>, and annotate it with when we started
+ * the test. */
+void
+dirserv_single_reachability_test(time_t now, routerinfo_t *router)
+{
+  const or_options_t *options = get_options();
+  channel_t *chan = NULL;
+  const node_t *node = NULL;
+  tor_addr_t router_addr;
+  const ed25519_public_key_t *ed_id_key;
+  (void) now;
+
+  tor_assert(router);
+  node = node_get_by_id(router->cache_info.identity_digest);
+  tor_assert(node);
+
+  if (options->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node, 1) &&
+      router->cache_info.signing_key_cert) {
+    ed_id_key = &router->cache_info.signing_key_cert->signing_key;
+  } else {
+    ed_id_key = NULL;
+  }
+
+  /* IPv4. */
+  log_debug(LD_OR,"Testing reachability of %s at %s:%u.",
+            router->nickname, fmt_addr32(router->addr), router->or_port);
+  tor_addr_from_ipv4h(&router_addr, router->addr);
+  chan = channel_tls_connect(&router_addr, router->or_port,
+                             router->cache_info.identity_digest,
+                             ed_id_key);
+  if (chan) command_setup_channel(chan);
+
+  /* Possible IPv6. */
+  if (get_options()->AuthDirHasIPv6Connectivity == 1 &&
+      !tor_addr_is_null(&router->ipv6_addr)) {
+    char addrstr[TOR_ADDR_BUF_LEN];
+    log_debug(LD_OR, "Testing reachability of %s at %s:%u.",
+              router->nickname,
+              tor_addr_to_str(addrstr, &router->ipv6_addr, sizeof(addrstr), 1),
+              router->ipv6_orport);
+    chan = channel_tls_connect(&router->ipv6_addr, router->ipv6_orport,
+                               router->cache_info.identity_digest,
+                               ed_id_key);
+    if (chan) command_setup_channel(chan);
+  }
+}
+
+/** Auth dir server only: load balance such that we only
+ * try a few connections per call.
+ *
+ * The load balancing is such that if we get called once every ten
+ * seconds, we will cycle through all the tests in
+ * REACHABILITY_TEST_CYCLE_PERIOD seconds (a bit over 20 minutes).
+ */
+void
+dirserv_test_reachability(time_t now)
+{
+  /* XXX decide what to do here; see or-talk thread "purging old router
+   * information, revocation." -NM
+   * We can't afford to mess with this in 0.1.2.x. The reason is that
+   * if we stop doing reachability tests on some of routerlist, then
+   * we'll for-sure think they're down, which may have unexpected
+   * effects in other parts of the code. It doesn't hurt much to do
+   * the testing, and directory authorities are easy to upgrade. Let's
+   * wait til 0.2.0. -RD */
+//  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  routerlist_t *rl = router_get_routerlist();
+  static char ctr = 0;
+  int bridge_auth = authdir_mode_bridge(get_options());
+
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, router) {
+    const char *id_digest = router->cache_info.identity_digest;
+    if (router_is_me(router))
+      continue;
+    if (bridge_auth && router->purpose != ROUTER_PURPOSE_BRIDGE)
+      continue; /* bridge authorities only test reachability on bridges */
+//    if (router->cache_info.published_on > cutoff)
+//      continue;
+    if ((((uint8_t)id_digest[0]) % REACHABILITY_MODULO_PER_TEST) == ctr) {
+      dirserv_single_reachability_test(now, router);
+    }
+  } SMARTLIST_FOREACH_END(router);
+  ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */
+}
+
+/* ==========
+ * Spooling code.
+ * ========== */
+
+spooled_resource_t *
+spooled_resource_new(dir_spool_source_t source,
+                     const uint8_t *digest, size_t digestlen)
+{
+  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
+  spooled->spool_source = source;
+  switch (source) {
+    case DIR_SPOOL_NETWORKSTATUS:
+      spooled->spool_eagerly = 0;
+      break;
+    case DIR_SPOOL_SERVER_BY_DIGEST:
+    case DIR_SPOOL_SERVER_BY_FP:
+    case DIR_SPOOL_EXTRA_BY_DIGEST:
+    case DIR_SPOOL_EXTRA_BY_FP:
+    case DIR_SPOOL_MICRODESC:
+    default:
+      spooled->spool_eagerly = 1;
+      break;
+    case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
+      tor_assert_unreached();
+      break;
+  }
+  tor_assert(digestlen <= sizeof(spooled->digest));
+  if (digest)
+    memcpy(spooled->digest, digest, digestlen);
+  return spooled;
+}
+
+/**
+ * Create a new spooled_resource_t to spool the contents of <b>entry</b> to
+ * the user.  Return the spooled object on success, or NULL on failure (which
+ * is probably caused by a failure to map the body of the item from disk).
+ *
+ * Adds a reference to entry's reference counter.
+ */
+spooled_resource_t *
+spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry)
+{
+  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
+  spooled->spool_source = DIR_SPOOL_CONSENSUS_CACHE_ENTRY;
+  spooled->spool_eagerly = 0;
+  consensus_cache_entry_incref(entry);
+  spooled->consensus_cache_entry = entry;
+
+  int r = consensus_cache_entry_get_body(entry,
+                                         &spooled->cce_body,
+                                         &spooled->cce_len);
+  if (r == 0) {
+    return spooled;
+  } else {
+    spooled_resource_free(spooled);
+    return NULL;
+  }
+}
+
+/** Release all storage held by <b>spooled</b>. */
+void
+spooled_resource_free_(spooled_resource_t *spooled)
+{
+  if (spooled == NULL)
+    return;
+
+  if (spooled->cached_dir_ref) {
+    cached_dir_decref(spooled->cached_dir_ref);
+  }
+
+  if (spooled->consensus_cache_entry) {
+    consensus_cache_entry_decref(spooled->consensus_cache_entry);
+  }
+
+  tor_free(spooled);
+}
+
+/** When spooling data from a cached_dir_t object, we always add
+ * at least this much. */
+#define DIRSERV_CACHED_DIR_CHUNK_SIZE 8192
+
+/** Return an compression ratio for compressing objects from <b>source</b>.
+ */
+static double
+estimate_compression_ratio(dir_spool_source_t source)
+{
+  /* We should put in better estimates here, depending on the number of
+     objects and their type */
+  (void) source;
+  return 0.5;
+}
+
+/** Return an estimated number of bytes needed for transmitting the
+ * resource in <b>spooled</b> on <b>conn</b>
+ *
+ * As a convenient side-effect, set *<b>published_out</b> to the resource's
+ * publication time.
+ */
+static size_t
+spooled_resource_estimate_size(const spooled_resource_t *spooled,
+                               dir_connection_t *conn,
+                               int compressed,
+                               time_t *published_out)
+{
+  if (spooled->spool_eagerly) {
+    const uint8_t *body = NULL;
+    size_t bodylen = 0;
+    int r = spooled_resource_lookup_body(spooled,
+                                         connection_dir_is_encrypted(conn),
+                                         &body, &bodylen,
+                                         published_out);
+    if (r == -1 || body == NULL || bodylen == 0)
+      return 0;
+    if (compressed) {
+      double ratio = estimate_compression_ratio(spooled->spool_source);
+      bodylen = (size_t)(bodylen * ratio);
+    }
+    return bodylen;
+  } else {
+    cached_dir_t *cached;
+    if (spooled->consensus_cache_entry) {
+      if (published_out) {
+        consensus_cache_entry_get_valid_after(
+            spooled->consensus_cache_entry, published_out);
+      }
+
+      return spooled->cce_len;
+    }
+    if (spooled->cached_dir_ref) {
+      cached = spooled->cached_dir_ref;
+    } else {
+      cached = spooled_resource_lookup_cached_dir(spooled,
+                                                  published_out);
+    }
+    if (cached == NULL) {
+      return 0;
+    }
+    size_t result = compressed ? cached->dir_compressed_len : cached->dir_len;
+    return result;
+  }
+}
+
+/** Return code for spooled_resource_flush_some */
+typedef enum {
+  SRFS_ERR = -1,
+  SRFS_MORE = 0,
+  SRFS_DONE
+} spooled_resource_flush_status_t;
+
+/** Flush some or all of the bytes from <b>spooled</b> onto <b>conn</b>.
+ * Return SRFS_ERR on error, SRFS_MORE if there are more bytes to flush from
+ * this spooled resource, or SRFS_DONE if we are done flushing this spooled
+ * resource.
+ */
+static spooled_resource_flush_status_t
+spooled_resource_flush_some(spooled_resource_t *spooled,
+                            dir_connection_t *conn)
+{
+  if (spooled->spool_eagerly) {
+    /* Spool_eagerly resources are sent all-at-once. */
+    const uint8_t *body = NULL;
+    size_t bodylen = 0;
+    int r = spooled_resource_lookup_body(spooled,
+                                         connection_dir_is_encrypted(conn),
+                                         &body, &bodylen, NULL);
+    if (r == -1 || body == NULL || bodylen == 0) {
+      /* Absent objects count as "done". */
+      return SRFS_DONE;
+    }
+    if (conn->compress_state) {
+      connection_buf_add_compress((const char*)body, bodylen, conn, 0);
+    } else {
+      connection_buf_add((const char*)body, bodylen, TO_CONN(conn));
+    }
+    return SRFS_DONE;
+  } else {
+    cached_dir_t *cached = spooled->cached_dir_ref;
+    consensus_cache_entry_t *cce = spooled->consensus_cache_entry;
+    if (cached == NULL && cce == NULL) {
+      /* The cached_dir_t hasn't been materialized yet. So let's look it up. */
+      cached = spooled->cached_dir_ref =
+        spooled_resource_lookup_cached_dir(spooled, NULL);
+      if (!cached) {
+        /* Absent objects count as done. */
+        return SRFS_DONE;
+      }
+      ++cached->refcnt;
+      tor_assert_nonfatal(spooled->cached_dir_offset == 0);
+    }
+
+    if (BUG(!cached && !cce))
+      return SRFS_DONE;
+
+    int64_t total_len;
+    const char *ptr;
+    if (cached) {
+      total_len = cached->dir_compressed_len;
+      ptr = cached->dir_compressed;
+    } else {
+      total_len = spooled->cce_len;
+      ptr = (const char *)spooled->cce_body;
+    }
+    /* How many bytes left to flush? */
+    int64_t remaining;
+    remaining = total_len - spooled->cached_dir_offset;
+    if (BUG(remaining < 0))
+      return SRFS_ERR;
+    ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
+    if (conn->compress_state) {
+      connection_buf_add_compress(
+              ptr + spooled->cached_dir_offset,
+              bytes, conn, 0);
+    } else {
+      connection_buf_add(ptr + spooled->cached_dir_offset,
+                              bytes, TO_CONN(conn));
+    }
+    spooled->cached_dir_offset += bytes;
+    if (spooled->cached_dir_offset >= (off_t)total_len) {
+      return SRFS_DONE;
+    } else {
+      return SRFS_MORE;
+    }
+  }
+}
+
+/** Helper: find the cached_dir_t for a spooled_resource_t, for
+ * sending it to <b>conn</b>. Set *<b>published_out</b>, if provided,
+ * to the published time of the cached_dir_t.
+ *
+ * DOES NOT increase the reference count on the result.  Callers must do that
+ * themselves if they mean to hang on to it.
+ */
+static cached_dir_t *
+spooled_resource_lookup_cached_dir(const spooled_resource_t *spooled,
+                                   time_t *published_out)
+{
+  tor_assert(spooled->spool_eagerly == 0);
+  cached_dir_t *d = lookup_cached_dir_by_fp(spooled->digest);
+  if (d != NULL) {
+    if (published_out)
+      *published_out = d->published;
+  }
+  return d;
+}
+
+/** Helper: Look up the body for an eagerly-served spooled_resource.  If
+ * <b>conn_is_encrypted</b> is false, don't look up any resource that
+ * shouldn't be sent over an unencrypted connection.  On success, set
+ * <b>body_out</b>, <b>size_out</b>, and <b>published_out</b> to refer
+ * to the resource's body, size, and publication date, and return 0.
+ * On failure return -1. */
+static int
+spooled_resource_lookup_body(const spooled_resource_t *spooled,
+                             int conn_is_encrypted,
+                             const uint8_t **body_out,
+                             size_t *size_out,
+                             time_t *published_out)
+{
+  tor_assert(spooled->spool_eagerly == 1);
+
+  const signed_descriptor_t *sd = NULL;
+
+  switch (spooled->spool_source) {
+    case DIR_SPOOL_EXTRA_BY_FP: {
+      sd = get_signed_descriptor_by_fp(spooled->digest, 1);
+      break;
+    }
+    case DIR_SPOOL_SERVER_BY_FP: {
+      sd = get_signed_descriptor_by_fp(spooled->digest, 0);
+      break;
+    }
+    case DIR_SPOOL_SERVER_BY_DIGEST: {
+      sd = router_get_by_descriptor_digest((const char *)spooled->digest);
+      break;
+    }
+    case DIR_SPOOL_EXTRA_BY_DIGEST: {
+      sd = extrainfo_get_by_descriptor_digest((const char *)spooled->digest);
+      break;
+    }
+    case DIR_SPOOL_MICRODESC: {
+      microdesc_t *md = microdesc_cache_lookup_by_digest256(
+                                  get_microdesc_cache(),
+                                  (const char *)spooled->digest);
+      if (! md || ! md->body) {
+        return -1;
+      }
+      *body_out = (const uint8_t *)md->body;
+      *size_out = md->bodylen;
+      if (published_out)
+        *published_out = TIME_MAX;
+      return 0;
+    }
+    case DIR_SPOOL_NETWORKSTATUS:
+    case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
+    default:
+      /* LCOV_EXCL_START */
+      tor_assert_nonfatal_unreached();
+      return -1;
+      /* LCOV_EXCL_STOP */
+  }
+
+  /* If we get here, then we tried to set "sd" to a signed_descriptor_t. */
+
+  if (sd == NULL) {
+    return -1;
+  }
+  if (sd->send_unencrypted == 0 && ! conn_is_encrypted) {
+    /* we did this check once before (so we could have an accurate size
+     * estimate and maybe send a 404 if somebody asked for only bridges on
+     * a connection), but we need to do it again in case a previously
+     * unknown bridge descriptor has shown up between then and now. */
+    return -1;
+  }
+  *body_out = (const uint8_t *) signed_descriptor_get_body(sd);
+  *size_out = sd->signed_descriptor_len;
+  if (published_out)
+    *published_out = sd->published_on;
+  return 0;
+}
+
+/** Given a fingerprint <b>fp</b> which is either set if we're looking for a
+ * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded
+ * flavor name if we want a flavored v3 status, return a pointer to the
+ * appropriate cached dir object, or NULL if there isn't one available. */
+static cached_dir_t *
+lookup_cached_dir_by_fp(const uint8_t *fp)
+{
+  cached_dir_t *d = NULL;
+  if (tor_digest_is_zero((const char *)fp) && cached_consensuses) {
+    d = strmap_get(cached_consensuses, "ns");
+  } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses) {
+    /* this here interface is a nasty hack: we're shoving a flavor into
+     * a digest field. */
+    d = strmap_get(cached_consensuses, (const char *)fp);
+  }
+  return d;
+}
+
+/** Try to guess the number of bytes that will be needed to send the
+ * spooled objects for <b>conn</b>'s outgoing spool.  In the process,
+ * remove every element of the spool that refers to an absent object, or
+ * which was published earlier than <b>cutoff</b>.  Set *<b>size_out</b>
+ * to the number of bytes, and *<b>n_expired_out</b> to the number of
+ * objects removed for being too old. */
+void
+dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn,
+                                            time_t cutoff,
+                                            int compression,
+                                            size_t *size_out,
+                                            int *n_expired_out)
+{
+  if (BUG(!conn))
+    return;
+
+  smartlist_t *spool = conn->spool;
+  if (!spool) {
+    if (size_out)
+      *size_out = 0;
+    if (n_expired_out)
+      *n_expired_out = 0;
+    return;
+  }
+  int n_expired = 0;
+  uint64_t total = 0;
+  SMARTLIST_FOREACH_BEGIN(spool, spooled_resource_t *, spooled) {
+    time_t published = TIME_MAX;
+    size_t sz = spooled_resource_estimate_size(spooled, conn,
+                                               compression, &published);
+    if (published < cutoff) {
+      ++n_expired;
+      SMARTLIST_DEL_CURRENT(spool, spooled);
+      spooled_resource_free(spooled);
+    } else if (sz == 0) {
+      SMARTLIST_DEL_CURRENT(spool, spooled);
+      spooled_resource_free(spooled);
+    } else {
+      total += sz;
+    }
+  } SMARTLIST_FOREACH_END(spooled);
+
+  if (size_out) {
+    *size_out = (total > SIZE_MAX) ? SIZE_MAX : (size_t)total;
+  }
+  if (n_expired_out)
+    *n_expired_out = n_expired;
+}
+
+/** Helper: used to sort a connection's spool. */
+static int
+dirserv_spool_sort_comparison_(const void **a_, const void **b_)
+{
+  const spooled_resource_t *a = *a_;
+  const spooled_resource_t *b = *b_;
+  return fast_memcmp(a->digest, b->digest, sizeof(a->digest));
+}
+
+/** Sort all the entries in <b>conn</b> by digest. */
+void
+dirserv_spool_sort(dir_connection_t *conn)
+{
+  if (conn->spool == NULL)
+    return;
+  smartlist_sort(conn->spool, dirserv_spool_sort_comparison_);
+}
+
+/** Return the cache-info for identity fingerprint <b>fp</b>, or
+ * its extra-info document if <b>extrainfo</b> is true. Return
+ * NULL if not found or if the descriptor is older than
+ * <b>publish_cutoff</b>. */
+static const signed_descriptor_t *
+get_signed_descriptor_by_fp(const uint8_t *fp, int extrainfo)
+{
+  if (router_digest_is_me((const char *)fp)) {
+    if (extrainfo)
+      return &(router_get_my_extrainfo()->cache_info);
+    else
+      return &(router_get_my_routerinfo()->cache_info);
+  } else {
+    const routerinfo_t *ri = router_get_by_id_digest((const char *)fp);
+    if (ri) {
+      if (extrainfo)
+        return extrainfo_get_by_descriptor_digest(
+                                     ri->cache_info.extra_info_digest);
+      else
+        return &ri->cache_info;
+    }
+  }
+  return NULL;
+}
+
+/** When we're spooling data onto our outbuf, add more whenever we dip
+ * below this threshold. */
+#define DIRSERV_BUFFER_MIN 16384
+
+/**
+ * Called whenever we have flushed some directory data in state
+ * SERVER_WRITING, or whenever we want to fill the buffer with initial
+ * directory data (so that subsequent writes will occur, and trigger this
+ * function again.)
+ *
+ * Return 0 on success, and -1 on failure.
+ */
+int
+connection_dirserv_flushed_some(dir_connection_t *conn)
+{
+  tor_assert(conn->base_.state == DIR_CONN_STATE_SERVER_WRITING);
+  if (conn->spool == NULL)
+    return 0;
+
+  while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN &&
+         smartlist_len(conn->spool)) {
+    spooled_resource_t *spooled =
+      smartlist_get(conn->spool, smartlist_len(conn->spool)-1);
+    spooled_resource_flush_status_t status;
+    status = spooled_resource_flush_some(spooled, conn);
+    if (status == SRFS_ERR) {
+      return -1;
+    } else if (status == SRFS_MORE) {
+      return 0;
+    }
+    tor_assert(status == SRFS_DONE);
+
+    /* If we're here, we're done flushing this resource. */
+    tor_assert(smartlist_pop_last(conn->spool) == spooled);
+    spooled_resource_free(spooled);
+  }
+
+  if (smartlist_len(conn->spool) > 0) {
+    /* We're still spooling something. */
+    return 0;
+  }
+
+  /* If we get here, we're done. */
+  smartlist_free(conn->spool);
+  conn->spool = NULL;
+  if (conn->compress_state) {
+    /* Flush the compression state: there could be more bytes pending in there,
+     * and we don't want to omit bytes. */
+    connection_buf_add_compress("", 0, conn, 1);
+    tor_compress_free(conn->compress_state);
+    conn->compress_state = NULL;
+  }
+  return 0;
+}
+
+/** Remove every element from <b>conn</b>'s outgoing spool, and delete
+ * the spool. */
+void
+dir_conn_clear_spool(dir_connection_t *conn)
+{
+  if (!conn || ! conn->spool)
+    return;
+  SMARTLIST_FOREACH(conn->spool, spooled_resource_t *, s,
+                    spooled_resource_free(s));
+  smartlist_free(conn->spool);
+  conn->spool = NULL;
+}
+
+/** Return true iff <b>line</b> is a valid RecommendedPackages line.
+ */
+/*
+  The grammar is:
+
+    "package" SP PACKAGENAME SP VERSION SP URL SP DIGESTS NL
+
+      PACKAGENAME = NONSPACE
+      VERSION = NONSPACE
+      URL = NONSPACE
+      DIGESTS = DIGEST | DIGESTS SP DIGEST
+      DIGEST = DIGESTTYPE "=" DIGESTVAL
+
+      NONSPACE = one or more non-space printing characters
+
+      DIGESTVAL = DIGESTTYPE = one or more non-=, non-" " characters.
+
+      SP = " "
+      NL = a newline
+
+ */
+int
+validate_recommended_package_line(const char *line)
+{
+  const char *cp = line;
+
+#define WORD()                                  \
+  do {                                          \
+    if (*cp == ' ')                             \
+      return 0;                                 \
+    cp = strchr(cp, ' ');                       \
+    if (!cp)                                    \
+      return 0;                                 \
+  } while (0)
+
+  WORD(); /* skip packagename */
+  ++cp;
+  WORD(); /* skip version */
+  ++cp;
+  WORD(); /* Skip URL */
+  ++cp;
+
+  /* Skip digesttype=digestval + */
+  int n_entries = 0;
+  while (1) {
+    const char *start_of_word = cp;
+    const char *end_of_word = strchr(cp, ' ');
+    if (! end_of_word)
+      end_of_word = cp + strlen(cp);
+
+    if (start_of_word == end_of_word)
+      return 0;
+
+    const char *eq = memchr(start_of_word, '=', end_of_word - start_of_word);
+
+    if (!eq)
+      return 0;
+    if (eq == start_of_word)
+      return 0;
+    if (eq == end_of_word - 1)
+      return 0;
+    if (memchr(eq+1, '=', end_of_word - (eq+1)))
+      return 0;
+
+    ++n_entries;
+    if (0 == *end_of_word)
+      break;
+
+    cp = end_of_word + 1;
+  }
+
+  /* If we reach this point, we have at least 1 entry. */
+  tor_assert(n_entries > 0);
+  return 1;
+}
+
+/** Release all storage used by the directory server. */
+void
+dirserv_free_all(void)
+{
+  dirserv_free_fingerprint_list();
+
+  strmap_free(cached_consensuses, free_cached_dir_);
+  cached_consensuses = NULL;
+
+  dirserv_clear_measured_bw_cache();
+}
diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h
new file mode 100644
index 0000000000..3b4a646094
--- /dev/null
+++ b/src/feature/dircache/dirserv.h
@@ -0,0 +1,239 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirserv.h
+ * \brief Header file for dirserv.c.
+ **/
+
+#ifndef TOR_DIRSERV_H
+#define TOR_DIRSERV_H
+
+struct ed25519_public_key_t;
+
+#include "lib/testsupport/testsupport.h"
+
+/** An enum to describe what format we're generating a routerstatus line in.
+ */
+typedef enum {
+  /** For use in a v2 opinion */
+  NS_V2,
+  /** For use in a consensus networkstatus document (ns flavor) */
+  NS_V3_CONSENSUS,
+  /** For use in a vote networkstatus document */
+  NS_V3_VOTE,
+  /** For passing to the controlport in response to a GETINFO request */
+  NS_CONTROL_PORT,
+  /** For use in a consensus networkstatus document (microdesc flavor) */
+  NS_V3_CONSENSUS_MICRODESC
+} routerstatus_format_type_t;
+
+/** What fraction (1 over this number) of the relay ID space do we
+ * (as a directory authority) launch connections to at each reachability
+ * test? */
+#define REACHABILITY_MODULO_PER_TEST 128
+
+/** How often (in seconds) do we launch reachability tests? */
+#define REACHABILITY_TEST_INTERVAL 10
+
+/** How many seconds apart are the reachability tests for a given relay? */
+#define REACHABILITY_TEST_CYCLE_PERIOD \
+  (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST)
+
+/** Maximum length of an exit policy summary. */
+#define MAX_EXITPOLICY_SUMMARY_LEN 1000
+
+/** Maximum allowable length of a version line in a networkstatus. */
+#define MAX_V_LINE_LEN 128
+
+/** Ways to convert a spoolable_resource_t to a bunch of bytes. */
+typedef enum dir_spool_source_t {
+    DIR_SPOOL_SERVER_BY_DIGEST=1, DIR_SPOOL_SERVER_BY_FP,
+    DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP,
+    DIR_SPOOL_MICRODESC,
+    DIR_SPOOL_NETWORKSTATUS,
+    DIR_SPOOL_CONSENSUS_CACHE_ENTRY,
+} dir_spool_source_t;
+#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t)
+
+/** Object to remember the identity of an object that we are spooling,
+ * or about to spool, in response to a directory request.
+ *
+ * (Why do we spool?  Because some directory responses are very large,
+ * and we don't want to just shove the complete answer into the output
+ * buffer: that would take a ridiculous amount of RAM.)
+ *
+ * If the spooled resource is relatively small (like microdescriptors,
+ * descriptors, etc), we look them up by ID as needed, and add the whole
+ * thing onto the output buffer at once.  If the spooled reseource is
+ * big (like networkstatus documents), we reference-count it, and add it
+ * a few K at a time.
+ */
+typedef struct spooled_resource_t {
+  /**
+   * If true, we add the entire object to the outbuf.  If false,
+   * we spool the object a few K at a time.
+   */
+  unsigned spool_eagerly : 1;
+  /**
+   * Tells us what kind of object to get, and how to look it up.
+   */
+  dir_spool_source_bitfield_t spool_source : 7;
+  /**
+   * Tells us the specific object to spool.
+   */
+  uint8_t digest[DIGEST256_LEN];
+  /**
+   * A large object that we're spooling. Holds a reference count.  Only
+   * used when spool_eagerly is false.
+   */
+  struct cached_dir_t *cached_dir_ref;
+  /**
+   * A different kind of large object that we might be spooling. Also
+   * reference-counted.  Also only used when spool_eagerly is false.
+   */
+  struct consensus_cache_entry_t *consensus_cache_entry;
+  const uint8_t *cce_body;
+  size_t cce_len;
+  /**
+   * The current offset into cached_dir or cce_body. Only used when
+   * spool_eagerly is false */
+  off_t cached_dir_offset;
+} spooled_resource_t;
+
+#ifdef DIRSERV_PRIVATE
+typedef struct measured_bw_line_t {
+  char node_id[DIGEST_LEN];
+  char node_hex[MAX_HEX_NICKNAME_LEN+1];
+  long int bw_kb;
+} measured_bw_line_t;
+#endif /* defined(DIRSERV_PRIVATE) */
+
+int connection_dirserv_flushed_some(dir_connection_t *conn);
+
+int dirserv_add_own_fingerprint(crypto_pk_t *pk);
+int dirserv_load_fingerprint_file(void);
+void dirserv_free_fingerprint_list(void);
+enum was_router_added_t dirserv_add_multiple_descriptors(
+                                     const char *desc, uint8_t purpose,
+                                     const char *source,
+                                     const char **msg);
+enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri,
+                                               const char **msg,
+                                               const char *source);
+void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
+int list_server_status_v1(smartlist_t *routers, char **router_status_out,
+                          int for_controller);
+char *dirserv_get_flag_thresholds_line(void);
+void dirserv_compute_bridge_flag_thresholds(void);
+
+int directory_fetches_from_authorities(const or_options_t *options);
+int directory_fetches_dir_info_early(const or_options_t *options);
+int directory_fetches_dir_info_later(const or_options_t *options);
+int directory_caches_unknown_auth_certs(const or_options_t *options);
+int directory_caches_dir_info(const or_options_t *options);
+int directory_permits_begindir_requests(const or_options_t *options);
+int directory_too_idle_to_fetch_descriptors(const or_options_t *options,
+                                            time_t now);
+
+cached_dir_t *dirserv_get_consensus(const char *flavor_name);
+void dirserv_set_cached_consensus_networkstatus(const char *consensus,
+                                              const char *flavor_name,
+                                              const common_digests_t *digests,
+                                              const uint8_t *sha3_as_signed,
+                                              time_t published);
+void dirserv_clear_old_networkstatuses(time_t cutoff);
+int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key,
+                                 dir_spool_source_t source,
+                                 int conn_is_encrypted,
+                                 const char **msg_out);
+int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
+                            const char **msg);
+void dirserv_orconn_tls_done(const tor_addr_t *addr,
+                             uint16_t or_port,
+                             const char *digest_rcvd,
+                             const struct ed25519_public_key_t *ed_id_rcvd);
+int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                            const routerinfo_t *ri_old);
+void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
+void dirserv_test_reachability(time_t now);
+int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                                   int complain,
+                                   int *valid_out);
+uint32_t dirserv_router_get_status(const routerinfo_t *router,
+                                   const char **msg,
+                                   int severity);
+void dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                      uint32_t authstatus);
+
+int dirserv_would_reject_router(const routerstatus_t *rs);
+char *routerstatus_format_entry(
+                              const routerstatus_t *rs,
+                              const char *version,
+                              const char *protocols,
+                              routerstatus_format_type_t format,
+                              int consensus_method,
+                              const vote_routerstatus_t *vrs);
+void dirserv_free_all(void);
+void cached_dir_decref(cached_dir_t *d);
+cached_dir_t *new_cached_dir(char *s, time_t published);
+
+int validate_recommended_package_line(const char *line);
+int dirserv_query_measured_bw_cache_kb(const char *node_id,
+                                       long *bw_out,
+                                       time_t *as_of_out);
+void dirserv_clear_measured_bw_cache(void);
+int dirserv_has_measured_bw(const char *node_id);
+int dirserv_get_measured_bw_cache_size(void);
+void dirserv_count_measured_bws(const smartlist_t *routers);
+int running_long_enough_to_decide_unreachable(void);
+void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil);
+
+#ifdef DIRSERV_PRIVATE
+
+STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
+
+/* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */
+#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */
+
+STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line,
+                                  int line_is_after_headers);
+
+STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line,
+                           smartlist_t *routerstatuses);
+
+STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
+                               time_t as_of);
+STATIC void dirserv_expire_measured_bw_cache(time_t now);
+
+STATIC int
+dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
+                                      smartlist_t *vote_routerstatuses);
+#endif /* defined(DIRSERV_PRIVATE) */
+
+int dirserv_read_measured_bandwidths(const char *from_file,
+                                     smartlist_t *routerstatuses);
+
+int dirserv_read_guardfraction_file(const char *fname,
+                                 smartlist_t *vote_routerstatuses);
+
+spooled_resource_t *spooled_resource_new(dir_spool_source_t source,
+                                         const uint8_t *digest,
+                                         size_t digestlen);
+spooled_resource_t *spooled_resource_new_from_cache_entry(
+                                      struct consensus_cache_entry_t *entry);
+void spooled_resource_free_(spooled_resource_t *spooled);
+#define spooled_resource_free(sp) \
+  FREE_AND_NULL(spooled_resource_t, spooled_resource_free_, (sp))
+void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn,
+                                                 time_t cutoff,
+                                                 int compression,
+                                                 size_t *size_out,
+                                                 int *n_expired_out);
+void dirserv_spool_sort(dir_connection_t *conn);
+void dir_conn_clear_spool(dir_connection_t *conn);
+
+#endif /* !defined(TOR_DIRSERV_H) */
diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h
new file mode 100644
index 0000000000..fe1f5c3d5f
--- /dev/null
+++ b/src/feature/dirclient/dir_server_st.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DIR_SERVER_ST_H
+#define DIR_SERVER_ST_H
+
+#include "lib/cc/torint.h"
+#include "or/or.h"
+#include "or/routerstatus_st.h"
+
+/** Represents information about a single trusted or fallback directory
+ * server. */
+struct dir_server_t {
+  char *description;
+  char *nickname;
+  char *address; /**< Hostname. */
+  /* XX/teor - why do we duplicate the address and port fields here and in
+   *           fake_status? Surely we could just use fake_status (#17867). */
+  tor_addr_t ipv6_addr; /**< IPv6 address if present; AF_UNSPEC if not */
+  uint32_t addr; /**< IPv4 address. */
+  uint16_t dir_port; /**< Directory port. */
+  uint16_t or_port; /**< OR port: Used for tunneling connections. */
+  uint16_t ipv6_orport; /**< OR port corresponding to ipv6_addr. */
+  double weight; /** Weight used when selecting this node at random */
+  char digest[DIGEST_LEN]; /**< Digest of identity key. */
+  char v3_identity_digest[DIGEST_LEN]; /**< Digest of v3 (authority only,
+                                        * high-security) identity key. */
+
+  unsigned int is_running:1; /**< True iff we think this server is running. */
+  unsigned int is_authority:1; /**< True iff this is a directory authority
+                                * of some kind. */
+
+  /** True iff this server has accepted the most recent server descriptor
+   * we tried to upload to it. */
+  unsigned int has_accepted_serverdesc:1;
+
+  /** What kind of authority is this? (Bitfield.) */
+  dirinfo_type_t type;
+
+  time_t addr_current_at; /**< When was the document that we derived the
+                           * address information from published? */
+
+  routerstatus_t fake_status; /**< Used when we need to pass this trusted
+                               * dir_server_t to
+                               * directory_request_set_routerstatus.
+                               * as a routerstatus_t.  Not updated by the
+                               * router-status management code!
+                               **/
+};
+
+#endif
diff --git a/src/feature/dirclient/download_status_st.h b/src/feature/dirclient/download_status_st.h
new file mode 100644
index 0000000000..3f18f754a1
--- /dev/null
+++ b/src/feature/dirclient/download_status_st.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DOWNLOAD_STATUS_ST_H
+#define DOWNLOAD_STATUS_ST_H
+
+/** Information about our plans for retrying downloads for a downloadable
+ * directory object.
+ * Each type of downloadable directory object has a corresponding retry
+ * <b>schedule</b>, which can be different depending on whether the object is
+ * being downloaded from an authority or a mirror (<b>want_authority</b>).
+ * <b>next_attempt_at</b> contains the next time we will attempt to download
+ * the object.
+ * For schedules that <b>increment_on</b> failure, <b>n_download_failures</b>
+ * is used to determine the position in the schedule. (Each schedule is a
+ * smartlist of integer delays, parsed from a CSV option.) Every time a
+ * connection attempt fails, <b>n_download_failures</b> is incremented,
+ * the new delay value is looked up from the schedule, and
+ * <b>next_attempt_at</b> is set delay seconds from the time the previous
+ * connection failed. Therefore, at most one failure-based connection can be
+ * in progress for each download_status_t.
+ * For schedules that <b>increment_on</b> attempt, <b>n_download_attempts</b>
+ * is used to determine the position in the schedule. Every time a
+ * connection attempt is made, <b>n_download_attempts</b> is incremented,
+ * the new delay value is looked up from the schedule, and
+ * <b>next_attempt_at</b> is set delay seconds from the time the previous
+ * connection was attempted. Therefore, multiple concurrent attempted-based
+ * connections can be in progress for each download_status_t.
+ * After an object is successfully downloaded, any other concurrent connections
+ * are terminated. A new schedule which starts at position 0 is used for
+ * subsequent downloads of the same object.
+ */
+struct download_status_t {
+  time_t next_attempt_at; /**< When should we try downloading this object
+                           * again? */
+  uint8_t n_download_failures; /**< Number of failed downloads of the most
+                                * recent object, since the last success. */
+  uint8_t n_download_attempts; /**< Number of (potentially concurrent) attempts
+                                * to download the most recent object, since
+                                * the last success. */
+  download_schedule_bitfield_t schedule : 8; /**< What kind of object is being
+                                              * downloaded? This determines the
+                                              * schedule used for the download.
+                                              */
+  download_want_authority_bitfield_t want_authority : 1; /**< Is the download
+                                              * happening from an authority
+                                              * or a mirror? This determines
+                                              * the schedule used for the
+                                              * download. */
+  download_schedule_increment_bitfield_t increment_on : 1; /**< does this
+                                        * schedule increment on each attempt,
+                                        * or after each failure? */
+  uint8_t last_backoff_position; /**< number of attempts/failures, depending
+                                  * on increment_on, when we last recalculated
+                                  * the delay.  Only updated if backoff
+                                  * == 1. */
+  int last_delay_used; /**< last delay used for random exponential backoff;
+                        * only updated if backoff == 1 */
+};
+
+#endif
+
diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c
new file mode 100644
index 0000000000..88edda716e
--- /dev/null
+++ b/src/feature/dircommon/consdiff.c
@@ -0,0 +1,1414 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file consdiff.c
+ * \brief Consensus diff implementation, including both the generation and the
+ * application of diffs in a minimal ed format.
+ *
+ * The consensus diff application is done in consdiff_apply_diff, which relies
+ * on apply_ed_diff for the main ed diff part and on some digest helper
+ * functions to check the digest hashes found in the consensus diff header.
+ *
+ * The consensus diff generation is more complex. consdiff_gen_diff generates
+ * it, relying on gen_ed_diff to generate the ed diff and some digest helper
+ * functions to generate the digest hashes.
+ *
+ * gen_ed_diff is the tricky bit. In it simplest form, it will take quadratic
+ * time and linear space to generate an ed diff given two smartlists. As shown
+ * in its comment section, calling calc_changes on the entire two consensuses
+ * will calculate what is to be added and what is to be deleted in the diff.
+ * Its comment section briefly explains how it works.
+ *
+ * In our case specific to consensuses, we take advantage of the fact that
+ * consensuses list routers sorted by their identities. We use that
+ * information to avoid running calc_changes on the whole smartlists.
+ * gen_ed_diff will navigate through the two consensuses identity by identity
+ * and will send small couples of slices to calc_changes, keeping the running
+ * time near-linear. This is explained in more detail in the gen_ed_diff
+ * comments.
+ *
+ * The allocation strategy tries to save time and memory by avoiding needless
+ * copies.  Instead of actually splitting the inputs into separate strings, we
+ * allocate cdline_t objects, each of which represents a line in the original
+ * object or in the output.  We use memarea_t allocators to manage the
+ * temporary memory we use when generating or applying diffs.
+ **/
+
+#define CONSDIFF_PRIVATE
+
+#include "or/or.h"
+#include "or/consdiff.h"
+#include "lib/memarea/memarea.h"
+#include "or/routerparse.h"
+
+static const char* ns_diff_version = "network-status-diff-version 1";
+static const char* hash_token = "hash";
+
+static char *consensus_join_lines(const smartlist_t *inp);
+
+/** Return true iff a and b have the same contents. */
+STATIC int
+lines_eq(const cdline_t *a, const cdline_t *b)
+{
+  return a->len == b->len && fast_memeq(a->s, b->s, a->len);
+}
+
+/** Return true iff a has the same contents as the nul-terminated string b. */
+STATIC int
+line_str_eq(const cdline_t *a, const char *b)
+{
+  const size_t len = strlen(b);
+  tor_assert(len <= UINT32_MAX);
+  cdline_t bline = { b, (uint32_t)len };
+  return lines_eq(a, &bline);
+}
+
+/** Return true iff a begins with the same contents as the nul-terminated
+ * string b. */
+static int
+line_starts_with_str(const cdline_t *a, const char *b)
+{
+  const size_t len = strlen(b);
+  tor_assert(len <= UINT32_MAX);
+  return a->len >= len && fast_memeq(a->s, b, len);
+}
+
+/** Return a new cdline_t holding as its contents the nul-terminated
+ * string s. Use the provided memory area for storage. */
+static cdline_t *
+cdline_linecpy(memarea_t *area, const char *s)
+{
+  size_t len = strlen(s);
+  const char *ss = memarea_memdup(area, s, len);
+  cdline_t *line = memarea_alloc(area, sizeof(cdline_t));
+  line->s = ss;
+  line->len = (uint32_t)len;
+  return line;
+}
+
+/** Add a cdline_t to <b>lst</b> holding as its contents the nul-terminated
+ * string s.  Use the provided memory area for storage. */
+STATIC void
+smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s)
+{
+  smartlist_add(lst, cdline_linecpy(area, s));
+}
+
+/** Compute the digest of <b>cons</b>, and store the result in
+ * <b>digest_out</b>. Return 0 on success, -1 on failure. */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_compute_digest,(const char *cons,
+                          consensus_digest_t *digest_out))
+{
+  int r = crypto_digest256((char*)digest_out->sha3_256,
+                           cons, strlen(cons), DIGEST_SHA3_256);
+  return r;
+}
+
+/** Compute the digest-as-signed of <b>cons</b>, and store the result in
+ * <b>digest_out</b>. Return 0 on success, -1 on failure. */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_compute_digest_as_signed,(const char *cons,
+                                    consensus_digest_t *digest_out))
+{
+  return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256,
+                                                    cons);
+}
+
+/** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_digest_eq,(const uint8_t *d1,
+                     const uint8_t *d2))
+{
+  return fast_memeq(d1, d2, DIGEST256_LEN);
+}
+
+/** Create (allocate) a new slice from a smartlist. Assumes that the start
+ * and the end indexes are within the bounds of the initial smartlist. The end
+ * element is not part of the resulting slice. If end is -1, the slice is to
+ * reach the end of the smartlist.
+ */
+STATIC smartlist_slice_t *
+smartlist_slice(const smartlist_t *list, int start, int end)
+{
+  int list_len = smartlist_len(list);
+  tor_assert(start >= 0);
+  tor_assert(start <= list_len);
+  if (end == -1) {
+    end = list_len;
+  }
+  tor_assert(start <= end);
+
+  smartlist_slice_t *slice = tor_malloc(sizeof(smartlist_slice_t));
+  slice->list = list;
+  slice->offset = start;
+  slice->len = end - start;
+  return slice;
+}
+
+/** Helper: Compute the longest common subsequence lengths for the two slices.
+ * Used as part of the diff generation to find the column at which to split
+ * slice2 while still having the optimal solution.
+ * If direction is -1, the navigation is reversed. Otherwise it must be 1.
+ * The length of the resulting integer array is that of the second slice plus
+ * one.
+ */
+STATIC int *
+lcs_lengths(const smartlist_slice_t *slice1, const smartlist_slice_t *slice2,
+            int direction)
+{
+  size_t a_size = sizeof(int) * (slice2->len+1);
+
+  /* Resulting lcs lengths. */
+  int *result = tor_malloc_zero(a_size);
+  /* Copy of the lcs lengths from the last iteration. */
+  int *prev = tor_malloc(a_size);
+
+  tor_assert(direction == 1 || direction == -1);
+
+  int si = slice1->offset;
+  if (direction == -1) {
+    si += (slice1->len-1);
+  }
+
+  for (int i = 0; i < slice1->len; ++i, si+=direction) {
+
+    const cdline_t *line1 = smartlist_get(slice1->list, si);
+    /* Store the last results. */
+    memcpy(prev, result, a_size);
+
+    int sj = slice2->offset;
+    if (direction == -1) {
+      sj += (slice2->len-1);
+    }
+
+    for (int j = 0; j < slice2->len; ++j, sj+=direction) {
+
+      const cdline_t *line2 = smartlist_get(slice2->list, sj);
+      if (lines_eq(line1, line2)) {
+        /* If the lines are equal, the lcs is one line longer. */
+        result[j + 1] = prev[j] + 1;
+      } else {
+        /* If not, see what lcs parent path is longer. */
+        result[j + 1] = MAX(result[j], prev[j + 1]);
+      }
+    }
+  }
+  tor_free(prev);
+  return result;
+}
+
+/** Helper: Trim any number of lines that are equally at the start or the end
+ * of both slices.
+ */
+STATIC void
+trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2)
+{
+  while (slice1->len>0 && slice2->len>0) {
+    const cdline_t *line1 = smartlist_get(slice1->list, slice1->offset);
+    const cdline_t *line2 = smartlist_get(slice2->list, slice2->offset);
+    if (!lines_eq(line1, line2)) {
+      break;
+    }
+    slice1->offset++; slice1->len--;
+    slice2->offset++; slice2->len--;
+  }
+
+  int i1 = (slice1->offset+slice1->len)-1;
+  int i2 = (slice2->offset+slice2->len)-1;
+
+  while (slice1->len>0 && slice2->len>0) {
+    const cdline_t *line1 = smartlist_get(slice1->list, i1);
+    const cdline_t *line2 = smartlist_get(slice2->list, i2);
+    if (!lines_eq(line1, line2)) {
+      break;
+    }
+    i1--;
+    slice1->len--;
+    i2--;
+    slice2->len--;
+  }
+}
+
+/** Like smartlist_string_pos, but uses a cdline_t, and is restricted to the
+ * bounds of the slice.
+ */
+STATIC int
+smartlist_slice_string_pos(const smartlist_slice_t *slice,
+                           const cdline_t *string)
+{
+  int end = slice->offset + slice->len;
+  for (int i = slice->offset; i < end; ++i) {
+    const cdline_t *el = smartlist_get(slice->list, i);
+    if (lines_eq(el, string)) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+/** Helper: Set all the appropriate changed booleans to true. The first slice
+ * must be of length 0 or 1. All the lines of slice1 and slice2 which are not
+ * present in the other slice will be set to changed in their bool array.
+ * The two changed bool arrays are passed in the same order as the slices.
+ */
+STATIC void
+set_changed(bitarray_t *changed1, bitarray_t *changed2,
+            const smartlist_slice_t *slice1, const smartlist_slice_t *slice2)
+{
+  int toskip = -1;
+  tor_assert(slice1->len == 0 || slice1->len == 1);
+
+  if (slice1->len == 1) {
+    const cdline_t *line_common = smartlist_get(slice1->list, slice1->offset);
+    toskip = smartlist_slice_string_pos(slice2, line_common);
+    if (toskip == -1) {
+      bitarray_set(changed1, slice1->offset);
+    }
+  }
+  int end = slice2->offset + slice2->len;
+  for (int i = slice2->offset; i < end; ++i) {
+    if (i != toskip) {
+      bitarray_set(changed2, i);
+    }
+  }
+}
+
+/*
+ * Helper: Given that slice1 has been split by half into top and bot, we want
+ * to fetch the column at which to split slice2 so that we are still on track
+ * to the optimal diff solution, i.e. the shortest one. We use lcs_lengths
+ * since the shortest diff is just another way to say the longest common
+ * subsequence.
+ */
+static int
+optimal_column_to_split(const smartlist_slice_t *top,
+                        const smartlist_slice_t *bot,
+                        const smartlist_slice_t *slice2)
+{
+  int *lens_top = lcs_lengths(top, slice2, 1);
+  int *lens_bot = lcs_lengths(bot, slice2, -1);
+  int column=0, max_sum=-1;
+
+  for (int i = 0; i < slice2->len+1; ++i) {
+    int sum = lens_top[i] + lens_bot[slice2->len-i];
+    if (sum > max_sum) {
+      column = i;
+      max_sum = sum;
+    }
+  }
+  tor_free(lens_top);
+  tor_free(lens_bot);
+
+  return column;
+}
+
+/**
+ * Helper: Figure out what elements are new or gone on the second smartlist
+ * relative to the first smartlist, and store the booleans in the bitarrays.
+ * True on the first bitarray means the element is gone, true on the second
+ * bitarray means it's new.
+ *
+ * In its base case, either of the smartlists is of length <= 1 and we can
+ * quickly see what elements are new or are gone. In the other case, we will
+ * split one smartlist by half and we'll use optimal_column_to_split to find
+ * the optimal column at which to split the second smartlist so that we are
+ * finding the smallest diff possible.
+ */
+STATIC void
+calc_changes(smartlist_slice_t *slice1,
+             smartlist_slice_t *slice2,
+             bitarray_t *changed1, bitarray_t *changed2)
+{
+  trim_slices(slice1, slice2);
+
+  if (slice1->len <= 1) {
+    set_changed(changed1, changed2, slice1, slice2);
+
+  } else if (slice2->len <= 1) {
+    set_changed(changed2, changed1, slice2, slice1);
+
+  /* Keep on splitting the slices in two. */
+  } else {
+    smartlist_slice_t *top, *bot, *left, *right;
+
+    /* Split the first slice in half. */
+    int mid = slice1->len/2;
+    top = smartlist_slice(slice1->list, slice1->offset, slice1->offset+mid);
+    bot = smartlist_slice(slice1->list, slice1->offset+mid,
+        slice1->offset+slice1->len);
+
+    /* Split the second slice by the optimal column. */
+    int mid2 = optimal_column_to_split(top, bot, slice2);
+    left = smartlist_slice(slice2->list, slice2->offset, slice2->offset+mid2);
+    right = smartlist_slice(slice2->list, slice2->offset+mid2,
+        slice2->offset+slice2->len);
+
+    calc_changes(top, left, changed1, changed2);
+    calc_changes(bot, right, changed1, changed2);
+    tor_free(top);
+    tor_free(bot);
+    tor_free(left);
+    tor_free(right);
+  }
+}
+
+/* This table is from crypto.c. The SP and PAD defines are different. */
+#define NOT_VALID_BASE64 255
+#define X NOT_VALID_BASE64
+#define SP NOT_VALID_BASE64
+#define PAD NOT_VALID_BASE64
+static const uint8_t base64_compare_table[256] = {
+  X, X, X, X, X, X, X, X, X, SP, SP, SP, X, SP, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  SP, X, X, X, X, X, X, X, X, X, X, 62, X, X, X, 63,
+  52, 53, 54, 55, 56, 57, 58, 59, 60, 61, X, X, X, PAD, X, X,
+  X, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+  15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, X, X, X, X, X,
+  X, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+  41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+};
+
+/** Helper: Get the identity hash from a router line, assuming that the line
+ * at least appears to be a router line and thus starts with "r ".
+ *
+ * If an identity hash is found, store it (without decoding it) in
+ * <b>hash_out</b>, and return 0.  On failure, return -1.
+ */
+STATIC int
+get_id_hash(const cdline_t *line, cdline_t *hash_out)
+{
+  if (line->len < 2)
+    return -1;
+
+  /* Skip the router name. */
+  const char *hash = memchr(line->s + 2, ' ', line->len - 2);
+  if (!hash) {
+    return -1;
+  }
+
+  hash++;
+  const char *hash_end = hash;
+  /* Stop when the first non-base64 character is found. Use unsigned chars to
+   * avoid negative indexes causing crashes.
+   */
+  while (base64_compare_table[*((unsigned char*)hash_end)]
+           != NOT_VALID_BASE64 &&
+         hash_end < line->s + line->len) {
+    hash_end++;
+  }
+
+  /* Empty hash. */
+  if (hash_end == hash) {
+    return -1;
+  }
+
+  hash_out->s = hash;
+  /* Always true because lines are limited to this length */
+  tor_assert(hash_end >= hash);
+  tor_assert((size_t)(hash_end - hash) <= UINT32_MAX);
+  hash_out->len = (uint32_t)(hash_end - hash);
+
+  return 0;
+}
+
+/** Helper: Check that a line is a valid router entry. We must at least be
+ * able to fetch a proper identity hash from it for it to be valid.
+ */
+STATIC int
+is_valid_router_entry(const cdline_t *line)
+{
+  if (line->len < 2 || fast_memneq(line->s, "r ", 2))
+    return 0;
+  cdline_t tmp;
+  return (get_id_hash(line, &tmp) == 0);
+}
+
+/** Helper: Find the next router line starting at the current position.
+ * Assumes that cur is lower than the length of the smartlist, i.e. it is a
+ * line within the bounds of the consensus. The only exception is when we
+ * don't want to skip the first line, in which case cur will be -1.
+ */
+STATIC int
+next_router(const smartlist_t *cons, int cur)
+{
+  int len = smartlist_len(cons);
+  tor_assert(cur >= -1 && cur < len);
+
+  if (++cur >= len) {
+    return len;
+  }
+
+  const cdline_t *line = smartlist_get(cons, cur);
+  while (!is_valid_router_entry(line)) {
+    if (++cur >= len) {
+      return len;
+    }
+    line = smartlist_get(cons, cur);
+  }
+  return cur;
+}
+
+/** Helper: compare two base64-encoded identity hashes, which may be of
+ * different lengths. Comparison ends when the first non-base64 char is found.
+ */
+STATIC int
+base64cmp(const cdline_t *hash1, const cdline_t *hash2)
+{
+  /* NULL is always lower, useful for last_hash which starts at NULL. */
+  if (!hash1->s && !hash2->s) {
+    return 0;
+  }
+  if (!hash1->s) {
+    return -1;
+  }
+  if (!hash2->s) {
+    return 1;
+  }
+
+  /* Don't index with a char; char may be signed. */
+  const unsigned char *a = (unsigned char*)hash1->s;
+  const unsigned char *b = (unsigned char*)hash2->s;
+  const unsigned char *a_end = a + hash1->len;
+  const unsigned char *b_end = b + hash2->len;
+  while (1) {
+    uint8_t av = base64_compare_table[*a];
+    uint8_t bv = base64_compare_table[*b];
+    if (av == NOT_VALID_BASE64) {
+      if (bv == NOT_VALID_BASE64) {
+        /* Both ended with exactly the same characters. */
+        return 0;
+      } else {
+        /* hash2 goes on longer than hash1 and thus hash1 is lower. */
+        return -1;
+      }
+    } else if (bv == NOT_VALID_BASE64) {
+      /* hash1 goes on longer than hash2 and thus hash1 is greater. */
+      return 1;
+    } else if (av < bv) {
+      /* The first difference shows that hash1 is lower. */
+      return -1;
+    } else if (av > bv) {
+      /* The first difference shows that hash1 is greater. */
+      return 1;
+    } else {
+      a++;
+      b++;
+      if (a == a_end) {
+        if (b == b_end) {
+          return 0;
+        } else {
+          return -1;
+        }
+      } else if (b == b_end) {
+        return 1;
+      }
+    }
+  }
+}
+
+/** Structure used to remember the previous and current identity hash of
+ * the "r " lines in a consensus, to enforce well-ordering. */
+typedef struct router_id_iterator_t {
+  cdline_t last_hash;
+  cdline_t hash;
+} router_id_iterator_t;
+
+/**
+ * Initializer for a router_id_iterator_t.
+ */
+#define ROUTER_ID_ITERATOR_INIT { { NULL, 0 }, { NULL, 0 } }
+
+/** Given an index *<b>idxp</b> into the consensus at <b>cons</b>, advance
+ * the index to the next router line ("r ...") in the consensus, or to
+ * an index one after the end of the list if there is no such line.
+ *
+ * Use <b>iter</b> to record the hash of the found router line, if any,
+ * and to enforce ordering on the hashes.  If the hashes are mis-ordered,
+ * return -1.  Else, return 0.
+ **/
+static int
+find_next_router_line(const smartlist_t *cons,
+                      const char *consname,
+                      int *idxp,
+                      router_id_iterator_t *iter)
+{
+  *idxp = next_router(cons, *idxp);
+  if (*idxp < smartlist_len(cons)) {
+    memcpy(&iter->last_hash, &iter->hash, sizeof(cdline_t));
+    if (get_id_hash(smartlist_get(cons, *idxp), &iter->hash) < 0 ||
+        base64cmp(&iter->hash, &iter->last_hash) <= 0) {
+      log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because "
+               "the %s consensus doesn't have its router entries sorted "
+               "properly.", consname);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+/** Line-prefix indicating the beginning of the signatures section that we
+ * intend to delete. */
+#define START_OF_SIGNATURES_SECTION "directory-signature "
+
+/** Pre-process a consensus in <b>cons</b> (represented as a list of cdline_t)
+ * to remove the signatures from it.  If the footer is removed, return a
+ * cdline_t containing a delete command to delete the footer, allocated in
+ * <b>area</>.  If no footer is removed, return NULL.
+ *
+ * We remove the signatures here because they are not themselves signed, and
+ * as such there might be different encodings for them.
+ */
+static cdline_t *
+preprocess_consensus(memarea_t *area,
+                     smartlist_t *cons)
+{
+  int idx;
+  int dirsig_idx = -1;
+  for (idx = 0; idx < smartlist_len(cons); ++idx) {
+    cdline_t *line = smartlist_get(cons, idx);
+    if (line_starts_with_str(line, START_OF_SIGNATURES_SECTION)) {
+      dirsig_idx = idx;
+      break;
+    }
+  }
+  if (dirsig_idx >= 0) {
+    char buf[64];
+    while (smartlist_len(cons) > dirsig_idx)
+      smartlist_del(cons, dirsig_idx);
+    tor_snprintf(buf, sizeof(buf), "%d,$d", dirsig_idx+1);
+    return cdline_linecpy(area, buf);
+  } else {
+    return NULL;
+  }
+}
+
+/** Generate an ed diff as a smartlist from two consensuses, also given as
+ * smartlists. Will return NULL if the diff could not be generated, which can
+ * happen if any lines the script had to add matched "." or if the routers
+ * were not properly ordered.
+ *
+ * All cdline_t objects in the resulting object are either references to lines
+ * in one of the inputs, or are newly allocated lines in the provided memarea.
+ *
+ * This implementation is consensus-specific. To generate an ed diff for any
+ * given input in quadratic time, you can replace all the code until the
+ * navigation in reverse order with the following:
+ *
+ *   int len1 = smartlist_len(cons1);
+ *   int len2 = smartlist_len(cons2);
+ *   bitarray_t *changed1 = bitarray_init_zero(len1);
+ *   bitarray_t *changed2 = bitarray_init_zero(len2);
+ *   cons1_sl = smartlist_slice(cons1, 0, -1);
+ *   cons2_sl = smartlist_slice(cons2, 0, -1);
+ *   calc_changes(cons1_sl, cons2_sl, changed1, changed2);
+ */
+STATIC smartlist_t *
+gen_ed_diff(const smartlist_t *cons1_orig, const smartlist_t *cons2,
+            memarea_t *area)
+{
+  smartlist_t *cons1 = smartlist_new();
+  smartlist_add_all(cons1, cons1_orig);
+  cdline_t *remove_trailer = preprocess_consensus(area, cons1);
+
+  int len1 = smartlist_len(cons1);
+  int len2 = smartlist_len(cons2);
+  smartlist_t *result = smartlist_new();
+
+  if (remove_trailer) {
+    /* There's a delete-the-trailer line at the end, so add it here. */
+    smartlist_add(result, remove_trailer);
+  }
+
+  /* Initialize the changed bitarrays to zero, so that calc_changes only needs
+   * to set the ones that matter and leave the rest untouched.
+   */
+  bitarray_t *changed1 = bitarray_init_zero(len1);
+  bitarray_t *changed2 = bitarray_init_zero(len2);
+  int i1=-1, i2=-1;
+  int start1=0, start2=0;
+
+  /* To check that hashes are ordered properly */
+  router_id_iterator_t iter1 = ROUTER_ID_ITERATOR_INIT;
+  router_id_iterator_t iter2 = ROUTER_ID_ITERATOR_INIT;
+
+  /* i1 and i2 are initialized at the first line of each consensus. They never
+   * reach past len1 and len2 respectively, since next_router doesn't let that
+   * happen. i1 and i2 are advanced by at least one line at each iteration as
+   * long as they have not yet reached len1 and len2, so the loop is
+   * guaranteed to end, and each pair of (i1,i2) will be inspected at most
+   * once.
+   */
+  while (i1 < len1 || i2 < len2) {
+
+    /* Advance each of the two navigation positions by one router entry if not
+     * yet at the end.
+     */
+    if (i1 < len1) {
+      if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) {
+        goto error_cleanup;
+      }
+    }
+
+    if (i2 < len2) {
+      if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) {
+        goto error_cleanup;
+      }
+    }
+
+    /* If we have reached the end of both consensuses, there is no need to
+     * compare hashes anymore, since this is the last iteration.
+     */
+    if (i1 < len1 || i2 < len2) {
+
+      /* Keep on advancing the lower (by identity hash sorting) position until
+       * we have two matching positions. The only other possible outcome is
+       * that a lower position reaches the end of the consensus before it can
+       * reach a hash that is no longer the lower one. Since there will always
+       * be a lower hash for as long as the loop runs, one of the two indexes
+       * will always be incremented, thus assuring that the loop must end
+       * after a finite number of iterations. If that cannot be because said
+       * consensus has already reached the end, both are extended to their
+       * respecting ends since we are done.
+       */
+      int cmp = base64cmp(&iter1.hash, &iter2.hash);
+      while (cmp != 0) {
+        if (i1 < len1 && cmp < 0) {
+          if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) {
+            goto error_cleanup;
+          }
+          if (i1 == len1) {
+            /* We finished the first consensus, so grab all the remaining
+             * lines of the second consensus and finish up.
+             */
+            i2 = len2;
+            break;
+          }
+        } else if (i2 < len2 && cmp > 0) {
+          if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) {
+            goto error_cleanup;
+          }
+          if (i2 == len2) {
+            /* We finished the second consensus, so grab all the remaining
+             * lines of the first consensus and finish up.
+             */
+            i1 = len1;
+            break;
+          }
+        } else {
+          i1 = len1;
+          i2 = len2;
+          break;
+        }
+        cmp = base64cmp(&iter1.hash, &iter2.hash);
+      }
+    }
+
+    /* Make slices out of these chunks (up to the common router entry) and
+     * calculate the changes for them.
+     * Error if any of the two slices are longer than 10K lines. That should
+     * never happen with any pair of real consensuses. Feeding more than 10K
+     * lines to calc_changes would be very slow anyway.
+     */
+#define MAX_LINE_COUNT (10000)
+    if (i1-start1 > MAX_LINE_COUNT || i2-start2 > MAX_LINE_COUNT) {
+      log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because "
+          "we found too few common router ids.");
+      goto error_cleanup;
+    }
+
+    smartlist_slice_t *cons1_sl = smartlist_slice(cons1, start1, i1);
+    smartlist_slice_t *cons2_sl = smartlist_slice(cons2, start2, i2);
+    calc_changes(cons1_sl, cons2_sl, changed1, changed2);
+    tor_free(cons1_sl);
+    tor_free(cons2_sl);
+    start1 = i1, start2 = i2;
+  }
+
+  /* Navigate the changes in reverse order and generate one ed command for
+   * each chunk of changes.
+   */
+  i1=len1-1, i2=len2-1;
+  char buf[128];
+  while (i1 >= 0 || i2 >= 0) {
+
+    int start1x, start2x, end1, end2, added, deleted;
+
+    /* We are at a point were no changed bools are true, so just keep going. */
+    if (!(i1 >= 0 && bitarray_is_set(changed1, i1)) &&
+        !(i2 >= 0 && bitarray_is_set(changed2, i2))) {
+      if (i1 >= 0) {
+        i1--;
+      }
+      if (i2 >= 0) {
+        i2--;
+      }
+      continue;
+    }
+
+    end1 = i1, end2 = i2;
+
+    /* Grab all contiguous changed lines */
+    while (i1 >= 0 && bitarray_is_set(changed1, i1)) {
+      i1--;
+    }
+    while (i2 >= 0 && bitarray_is_set(changed2, i2)) {
+      i2--;
+    }
+
+    start1x = i1+1, start2x = i2+1;
+    added = end2-i2, deleted = end1-i1;
+
+    if (added == 0) {
+      if (deleted == 1) {
+        tor_snprintf(buf, sizeof(buf), "%id", start1x+1);
+        smartlist_add_linecpy(result, area, buf);
+      } else {
+        tor_snprintf(buf, sizeof(buf), "%i,%id", start1x+1, start1x+deleted);
+        smartlist_add_linecpy(result, area, buf);
+      }
+    } else {
+      int i;
+      if (deleted == 0) {
+        tor_snprintf(buf, sizeof(buf), "%ia", start1x);
+        smartlist_add_linecpy(result, area, buf);
+      } else if (deleted == 1) {
+        tor_snprintf(buf, sizeof(buf), "%ic", start1x+1);
+        smartlist_add_linecpy(result, area, buf);
+      } else {
+        tor_snprintf(buf, sizeof(buf), "%i,%ic", start1x+1, start1x+deleted);
+        smartlist_add_linecpy(result, area, buf);
+      }
+
+      for (i = start2x; i <= end2; ++i) {
+        cdline_t *line = smartlist_get(cons2, i);
+        if (line_str_eq(line, ".")) {
+          log_warn(LD_CONSDIFF, "Cannot generate consensus diff because "
+              "one of the lines to be added is \".\".");
+          goto error_cleanup;
+        }
+        smartlist_add(result, line);
+      }
+      smartlist_add_linecpy(result, area, ".");
+    }
+  }
+
+  smartlist_free(cons1);
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+
+  return result;
+
+ error_cleanup:
+
+  smartlist_free(cons1);
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+
+  smartlist_free(result);
+
+  return NULL;
+}
+
+/* Helper: Read a base-10 number between 0 and INT32_MAX from <b>s</b> and
+ * store it in <b>num_out</b>.  Advance <b>s</b> to the characer immediately
+ * after the number.  Return 0 on success, -1 on failure. */
+static int
+get_linenum(const char **s, int *num_out)
+{
+  int ok;
+  char *next;
+  if (!TOR_ISDIGIT(**s)) {
+    return -1;
+  }
+  *num_out = (int) tor_parse_long(*s, 10, 0, INT32_MAX, &ok, &next);
+  if (ok && next) {
+    *s = next;
+    return 0;
+  } else {
+    return -1;
+  }
+}
+
+/** Apply the ed diff, starting at <b>diff_starting_line</b>, to the consensus
+ * and return a new consensus, also as a line-based smartlist. Will return
+ * NULL if the ed diff is not properly formatted.
+ *
+ * All cdline_t objects in the resulting object are references to lines
+ * in one of the inputs; nothing is copied.
+ */
+STATIC smartlist_t *
+apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff,
+              int diff_starting_line)
+{
+  int diff_len = smartlist_len(diff);
+  int j = smartlist_len(cons1);
+  smartlist_t *cons2 = smartlist_new();
+
+  for (int i=diff_starting_line; i<diff_len; ++i) {
+    const cdline_t *diff_cdline = smartlist_get(diff, i);
+    char diff_line[128];
+
+    if (diff_cdline->len > sizeof(diff_line) - 1) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "an ed command was far too long");
+      goto error_cleanup;
+    }
+    /* Copy the line to make it nul-terminated. */
+    memcpy(diff_line, diff_cdline->s, diff_cdline->len);
+    diff_line[diff_cdline->len] = 0;
+    const char *ptr = diff_line;
+    int start = 0, end = 0;
+    int had_range = 0;
+    int end_was_eof = 0;
+    if (get_linenum(&ptr, &start) < 0) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "an ed command was missing a line number.");
+      goto error_cleanup;
+    }
+    if (*ptr == ',') {
+      /* Two-item range */
+      had_range = 1;
+      ++ptr;
+      if (*ptr == '$') {
+        end_was_eof = 1;
+        end = smartlist_len(cons1);
+        ++ptr;
+      } else if (get_linenum(&ptr, &end) < 0) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+                 "an ed command was missing a range end line number.");
+        goto error_cleanup;
+      }
+      /* Incoherent range. */
+      if (end <= start) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+                 "an invalid range was found in an ed command.");
+        goto error_cleanup;
+      }
+    } else {
+      /* We'll take <n1> as <n1>,<n1> for simplicity. */
+      end = start;
+    }
+
+    if (end > j) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "its commands are not properly sorted in reverse order.");
+      goto error_cleanup;
+    }
+
+    if (*ptr == '\0') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "a line with no ed command was found");
+      goto error_cleanup;
+    }
+
+    if (*(ptr+1) != '\0') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "an ed command longer than one char was found.");
+      goto error_cleanup;
+    }
+
+    char action = *ptr;
+
+    switch (action) {
+      case 'a':
+      case 'c':
+      case 'd':
+        break;
+      default:
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+            "an unrecognised ed command was found.");
+        goto error_cleanup;
+    }
+
+    /** $ is not allowed with non-d actions. */
+    if (end_was_eof && action != 'd') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "it wanted to use $ with a command other than delete");
+      goto error_cleanup;
+    }
+
+    /* 'a' commands are not allowed to have ranges. */
+    if (had_range && action == 'a') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "it wanted to add lines after a range.");
+      goto error_cleanup;
+    }
+
+    /* Add unchanged lines. */
+    for (; j && j > end; --j) {
+      cdline_t *cons_line = smartlist_get(cons1, j-1);
+      smartlist_add(cons2, cons_line);
+    }
+
+    /* Ignore removed lines. */
+    if (action == 'c' || action == 'd') {
+      while (--j >= start) {
+        /* Skip line */
+      }
+    }
+
+    /* Add new lines in reverse order, since it will all be reversed at the
+     * end.
+     */
+    if (action == 'a' || action == 'c') {
+      int added_end = i;
+
+      i++; /* Skip the line with the range and command. */
+      while (i < diff_len) {
+        if (line_str_eq(smartlist_get(diff, i), ".")) {
+          break;
+        }
+        if (++i == diff_len) {
+          log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+              "it has lines to be inserted that don't end with a \".\".");
+          goto error_cleanup;
+        }
+      }
+
+      int added_i = i-1;
+
+      /* It would make no sense to add zero new lines. */
+      if (added_i == added_end) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+            "it has an ed command that tries to insert zero lines.");
+        goto error_cleanup;
+      }
+
+      while (added_i > added_end) {
+        cdline_t *added_line = smartlist_get(diff, added_i--);
+        smartlist_add(cons2, added_line);
+      }
+    }
+  }
+
+  /* Add remaining unchanged lines. */
+  for (; j > 0; --j) {
+    cdline_t *cons_line = smartlist_get(cons1, j-1);
+    smartlist_add(cons2, cons_line);
+  }
+
+  /* Reverse the whole thing since we did it from the end. */
+  smartlist_reverse(cons2);
+  return cons2;
+
+ error_cleanup:
+
+  smartlist_free(cons2);
+
+  return NULL;
+}
+
+/** Generate a consensus diff as a smartlist from two given consensuses, also
+ * as smartlists. Will return NULL if the consensus diff could not be
+ * generated. Neither of the two consensuses are modified in any way, so it's
+ * up to the caller to free their resources.
+ */
+smartlist_t *
+consdiff_gen_diff(const smartlist_t *cons1,
+                  const smartlist_t *cons2,
+                  const consensus_digest_t *digests1,
+                  const consensus_digest_t *digests2,
+                  memarea_t *area)
+{
+  smartlist_t *ed_diff = gen_ed_diff(cons1, cons2, area);
+  /* ed diff could not be generated - reason already logged by gen_ed_diff. */
+  if (!ed_diff) {
+    goto error_cleanup;
+  }
+
+  /* See that the script actually produces what we want. */
+  smartlist_t *ed_cons2 = apply_ed_diff(cons1, ed_diff, 0);
+  if (!ed_cons2) {
+    /* LCOV_EXCL_START -- impossible if diff generation is correct */
+    log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because "
+        "the generated ed diff could not be tested to successfully generate "
+        "the target consensus.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  int cons2_eq = 1;
+  if (smartlist_len(cons2) == smartlist_len(ed_cons2)) {
+    SMARTLIST_FOREACH_BEGIN(cons2, const cdline_t *, line1) {
+      const cdline_t *line2 = smartlist_get(ed_cons2, line1_sl_idx);
+      if (! lines_eq(line1, line2) ) {
+        cons2_eq = 0;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(line1);
+  } else {
+    cons2_eq = 0;
+  }
+  smartlist_free(ed_cons2);
+  if (!cons2_eq) {
+    /* LCOV_EXCL_START -- impossible if diff generation is correct. */
+    log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because "
+        "the generated ed diff did not generate the target consensus "
+        "successfully when tested.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  char cons1_hash_hex[HEX_DIGEST256_LEN+1];
+  char cons2_hash_hex[HEX_DIGEST256_LEN+1];
+  base16_encode(cons1_hash_hex, HEX_DIGEST256_LEN+1,
+                (const char*)digests1->sha3_256, DIGEST256_LEN);
+  base16_encode(cons2_hash_hex, HEX_DIGEST256_LEN+1,
+                (const char*)digests2->sha3_256, DIGEST256_LEN);
+
+  /* Create the resulting consensus diff. */
+  char buf[160];
+  smartlist_t *result = smartlist_new();
+  tor_snprintf(buf, sizeof(buf), "%s", ns_diff_version);
+  smartlist_add_linecpy(result, area, buf);
+  tor_snprintf(buf, sizeof(buf), "%s %s %s", hash_token,
+      cons1_hash_hex, cons2_hash_hex);
+  smartlist_add_linecpy(result, area, buf);
+  smartlist_add_all(result, ed_diff);
+  smartlist_free(ed_diff);
+  return result;
+
+ error_cleanup:
+
+  if (ed_diff) {
+    /* LCOV_EXCL_START -- ed_diff is NULL except in unreachable cases above */
+    smartlist_free(ed_diff);
+    /* LCOV_EXCL_STOP */
+  }
+
+  return NULL;
+}
+
+/** Fetch the digest of the base consensus in the consensus diff, encoded in
+ * base16 as found in the diff itself. digest1_out and digest2_out must be of
+ * length DIGEST256_LEN or larger if not NULL.
+ */
+int
+consdiff_get_digests(const smartlist_t *diff,
+                     char *digest1_out,
+                     char *digest2_out)
+{
+  smartlist_t *hash_words = NULL;
+  const cdline_t *format;
+  char cons1_hash[DIGEST256_LEN], cons2_hash[DIGEST256_LEN];
+  char *cons1_hash_hex, *cons2_hash_hex;
+  if (smartlist_len(diff) < 2) {
+    log_info(LD_CONSDIFF, "The provided consensus diff is too short.");
+    goto error_cleanup;
+  }
+
+  /* Check that it's the format and version we know. */
+  format = smartlist_get(diff, 0);
+  if (!line_str_eq(format, ns_diff_version)) {
+    log_warn(LD_CONSDIFF, "The provided consensus diff format is not known.");
+    goto error_cleanup;
+  }
+
+  /* Grab the base16 digests. */
+  hash_words = smartlist_new();
+  {
+    const cdline_t *line2 = smartlist_get(diff, 1);
+    char *h = tor_memdup_nulterm(line2->s, line2->len);
+    smartlist_split_string(hash_words, h, " ", 0, 0);
+    tor_free(h);
+  }
+
+  /* There have to be three words, the first of which must be hash_token. */
+  if (smartlist_len(hash_words) != 3 ||
+      strcmp(smartlist_get(hash_words, 0), hash_token)) {
+    log_info(LD_CONSDIFF, "The provided consensus diff does not include "
+        "the necessary digests.");
+    goto error_cleanup;
+  }
+
+  /* Expected hashes as found in the consensus diff header. They must be of
+   * length HEX_DIGEST256_LEN, normally 64 hexadecimal characters.
+   * If any of the decodings fail, error to make sure that the hashes are
+   * proper base16-encoded digests.
+   */
+  cons1_hash_hex = smartlist_get(hash_words, 1);
+  cons2_hash_hex = smartlist_get(hash_words, 2);
+  if (strlen(cons1_hash_hex) != HEX_DIGEST256_LEN ||
+      strlen(cons2_hash_hex) != HEX_DIGEST256_LEN) {
+    log_info(LD_CONSDIFF, "The provided consensus diff includes "
+        "base16-encoded digests of incorrect size.");
+    goto error_cleanup;
+  }
+
+  if (base16_decode(cons1_hash, DIGEST256_LEN,
+          cons1_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN ||
+      base16_decode(cons2_hash, DIGEST256_LEN,
+          cons2_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN) {
+    log_info(LD_CONSDIFF, "The provided consensus diff includes "
+        "malformed digests.");
+    goto error_cleanup;
+  }
+
+  if (digest1_out) {
+    memcpy(digest1_out, cons1_hash, DIGEST256_LEN);
+  }
+  if (digest2_out) {
+    memcpy(digest2_out, cons2_hash, DIGEST256_LEN);
+  }
+
+  SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp));
+  smartlist_free(hash_words);
+  return 0;
+
+ error_cleanup:
+
+  if (hash_words) {
+    SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp));
+    smartlist_free(hash_words);
+  }
+  return 1;
+}
+
+/** Apply the consensus diff to the given consensus and return a new
+ * consensus, also as a line-based smartlist. Will return NULL if the diff
+ * could not be applied. Neither the consensus nor the diff are modified in
+ * any way, so it's up to the caller to free their resources.
+ */
+char *
+consdiff_apply_diff(const smartlist_t *cons1,
+                    const smartlist_t *diff,
+                    const consensus_digest_t *digests1)
+{
+  smartlist_t *cons2 = NULL;
+  char *cons2_str = NULL;
+  char e_cons1_hash[DIGEST256_LEN];
+  char e_cons2_hash[DIGEST256_LEN];
+
+  if (consdiff_get_digests(diff, e_cons1_hash, e_cons2_hash) != 0) {
+    goto error_cleanup;
+  }
+
+  /* See that the consensus that was given to us matches its hash. */
+  if (!consensus_digest_eq(digests1->sha3_256,
+                           (const uint8_t*)e_cons1_hash)) {
+    char hex_digest1[HEX_DIGEST256_LEN+1];
+    char e_hex_digest1[HEX_DIGEST256_LEN+1];
+    log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because "
+        "the base consensus doesn't match the digest as found in "
+        "the consensus diff header.");
+    base16_encode(hex_digest1, HEX_DIGEST256_LEN+1,
+                  (const char *)digests1->sha3_256, DIGEST256_LEN);
+    base16_encode(e_hex_digest1, HEX_DIGEST256_LEN+1,
+                  e_cons1_hash, DIGEST256_LEN);
+    log_warn(LD_CONSDIFF, "Expected: %s; found: %s",
+             hex_digest1, e_hex_digest1);
+    goto error_cleanup;
+  }
+
+  /* Grab the ed diff and calculate the resulting consensus. */
+  /* Skip the first two lines. */
+  cons2 = apply_ed_diff(cons1, diff, 2);
+
+  /* ed diff could not be applied - reason already logged by apply_ed_diff. */
+  if (!cons2) {
+    goto error_cleanup;
+  }
+
+  cons2_str = consensus_join_lines(cons2);
+
+  consensus_digest_t cons2_digests;
+  if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) {
+    /* LCOV_EXCL_START -- digest can't fail */
+    log_warn(LD_CONSDIFF, "Could not compute digests of the consensus "
+        "resulting from applying a consensus diff.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* See that the resulting consensus matches its hash. */
+  if (!consensus_digest_eq(cons2_digests.sha3_256,
+                           (const uint8_t*)e_cons2_hash)) {
+    log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because "
+        "the resulting consensus doesn't match the digest as found in "
+        "the consensus diff header.");
+    char hex_digest2[HEX_DIGEST256_LEN+1];
+    char e_hex_digest2[HEX_DIGEST256_LEN+1];
+    base16_encode(hex_digest2, HEX_DIGEST256_LEN+1,
+        (const char *)cons2_digests.sha3_256, DIGEST256_LEN);
+    base16_encode(e_hex_digest2, HEX_DIGEST256_LEN+1,
+        e_cons2_hash, DIGEST256_LEN);
+    log_warn(LD_CONSDIFF, "Expected: %s; found: %s",
+             hex_digest2, e_hex_digest2);
+    goto error_cleanup;
+  }
+
+  goto done;
+
+ error_cleanup:
+  tor_free(cons2_str); /* Sets it to NULL */
+
+ done:
+  if (cons2) {
+    smartlist_free(cons2);
+  }
+
+  return cons2_str;
+}
+
+/** Any consensus line longer than this means that the input is invalid. */
+#define CONSENSUS_LINE_MAX_LEN (1<<20)
+
+/**
+ * Helper: For every NL-terminated line in <b>s</b>, add a cdline referring to
+ * that line (without trailing newline) to <b>out</b>.  Return -1 if there are
+ * any non-NL terminated lines; 0 otherwise.
+ *
+ * Unlike tor_split_lines, this function avoids ambiguity on its
+ * handling of a final line that isn't NL-terminated.
+ *
+ * All cdline_t objects are allocated in the provided memarea.  Strings
+ * are not copied: if <b>s</b> changes or becomes invalid, then all
+ * generated cdlines will become invalid.
+ */
+STATIC int
+consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area)
+{
+  const char *end_of_str = s + strlen(s);
+  tor_assert(*end_of_str == '\0');
+
+  while (*s) {
+    const char *eol = memchr(s, '\n', end_of_str - s);
+    if (!eol) {
+      /* File doesn't end with newline. */
+      return -1;
+    }
+    if (eol - s > CONSENSUS_LINE_MAX_LEN) {
+      /* Line is far too long. */
+      return -1;
+    }
+    cdline_t *line = memarea_alloc(area, sizeof(cdline_t));
+    line->s = s;
+    line->len = (uint32_t)(eol - s);
+    smartlist_add(out, line);
+    s = eol+1;
+  }
+  return 0;
+}
+
+/** Given a list of cdline_t, return a newly allocated string containing
+ * all of the lines, terminated with NL, concatenated.
+ *
+ * Unlike smartlist_join_strings(), avoids lossy operations on empty
+ * lists.  */
+static char *
+consensus_join_lines(const smartlist_t *inp)
+{
+  size_t n = 0;
+  SMARTLIST_FOREACH(inp, const cdline_t *, cdline, n += cdline->len + 1);
+  n += 1;
+  char *result = tor_malloc(n);
+  char *out = result;
+  SMARTLIST_FOREACH_BEGIN(inp, const cdline_t *, cdline) {
+    memcpy(out, cdline->s, cdline->len);
+    out += cdline->len;
+    *out++ = '\n';
+  } SMARTLIST_FOREACH_END(cdline);
+  *out++ = '\0';
+  tor_assert(out == result+n);
+  return result;
+}
+
+/** Given two consensus documents, try to compute a diff between them.  On
+ * success, retun a newly allocated string containing that diff.  On failure,
+ * return NULL. */
+char *
+consensus_diff_generate(const char *cons1,
+                        const char *cons2)
+{
+  consensus_digest_t d1, d2;
+  smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL;
+  int r1, r2;
+  char *result = NULL;
+
+  r1 = consensus_compute_digest_as_signed(cons1, &d1);
+  r2 = consensus_compute_digest(cons2, &d2);
+  if (BUG(r1 < 0 || r2 < 0))
+    return NULL; // LCOV_EXCL_LINE
+
+  memarea_t *area = memarea_new();
+  lines1 = smartlist_new();
+  lines2 = smartlist_new();
+  if (consensus_split_lines(lines1, cons1, area) < 0)
+    goto done;
+  if (consensus_split_lines(lines2, cons2, area) < 0)
+    goto done;
+
+  result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area);
+
+ done:
+  if (result_lines) {
+    result = consensus_join_lines(result_lines);
+    smartlist_free(result_lines);
+  }
+
+  memarea_drop_all(area);
+  smartlist_free(lines1);
+  smartlist_free(lines2);
+
+  return result;
+}
+
+/** Given a consensus document and a diff, try to apply the diff to the
+ * consensus.  On success return a newly allocated string containing the new
+ * consensus.  On failure, return NULL. */
+char *
+consensus_diff_apply(const char *consensus,
+                     const char *diff)
+{
+  consensus_digest_t d1;
+  smartlist_t *lines1 = NULL, *lines2 = NULL;
+  int r1;
+  char *result = NULL;
+  memarea_t *area = memarea_new();
+
+  r1 = consensus_compute_digest_as_signed(consensus, &d1);
+  if (BUG(r1 < 0))
+    return NULL; // LCOV_EXCL_LINE
+
+  lines1 = smartlist_new();
+  lines2 = smartlist_new();
+  if (consensus_split_lines(lines1, consensus, area) < 0)
+    goto done;
+  if (consensus_split_lines(lines2, diff, area) < 0)
+    goto done;
+
+  result = consdiff_apply_diff(lines1, lines2, &d1);
+
+ done:
+  smartlist_free(lines1);
+  smartlist_free(lines2);
+  memarea_drop_all(area);
+
+  return result;
+}
+
+/** Return true iff, based on its header, <b>document</b> is likely
+ * to be a consensus diff. */
+int
+looks_like_a_consensus_diff(const char *document, size_t len)
+{
+  return (len >= strlen(ns_diff_version) &&
+          fast_memeq(document, ns_diff_version, strlen(ns_diff_version)));
+}
diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h
new file mode 100644
index 0000000000..1cae59a1a5
--- /dev/null
+++ b/src/feature/dircommon/consdiff.h
@@ -0,0 +1,99 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSDIFF_H
+#define TOR_CONSDIFF_H
+
+#include "or/or.h"
+
+char *consensus_diff_generate(const char *cons1,
+                              const char *cons2);
+char *consensus_diff_apply(const char *consensus,
+                           const char *diff);
+
+int looks_like_a_consensus_diff(const char *document, size_t len);
+
+#ifdef CONSDIFF_PRIVATE
+#include "lib/container/bitarray.h"
+
+struct memarea_t;
+
+/** Line type used for constructing consensus diffs.  Each of these lines
+ * refers to a chunk of memory allocated elsewhere, and is not necessarily
+ * NUL-terminated: this helps us avoid copies and save memory. */
+typedef struct cdline_t {
+  const char *s;
+  uint32_t len;
+} cdline_t;
+
+typedef struct consensus_digest_t {
+  uint8_t sha3_256[DIGEST256_LEN];
+} consensus_digest_t;
+
+STATIC smartlist_t *consdiff_gen_diff(const smartlist_t *cons1,
+                                      const smartlist_t *cons2,
+                                      const consensus_digest_t *digests1,
+                                      const consensus_digest_t *digests2,
+                                      struct memarea_t *area);
+STATIC char *consdiff_apply_diff(const smartlist_t *cons1,
+                                 const smartlist_t *diff,
+                                 const consensus_digest_t *digests1);
+STATIC int consdiff_get_digests(const smartlist_t *diff,
+                                char *digest1_out,
+                                char *digest2_out);
+
+/** Data structure to define a slice of a smarltist. */
+typedef struct smartlist_slice_t {
+  /**
+   * Smartlist that this slice is made from.
+   * References the whole original smartlist that the slice was made out of.
+   * */
+  const smartlist_t *list;
+  /** Starting position of the slice in the smartlist. */
+  int offset;
+  /** Length of the slice, i.e. the number of elements it holds. */
+  int len;
+} smartlist_slice_t;
+STATIC smartlist_t *gen_ed_diff(const smartlist_t *cons1,
+                                const smartlist_t *cons2,
+                                struct memarea_t *area);
+STATIC smartlist_t *apply_ed_diff(const smartlist_t *cons1,
+                                  const smartlist_t *diff,
+                                  int start_line);
+STATIC void calc_changes(smartlist_slice_t *slice1, smartlist_slice_t *slice2,
+                         bitarray_t *changed1, bitarray_t *changed2);
+STATIC smartlist_slice_t *smartlist_slice(const smartlist_t *list,
+                                          int start, int end);
+STATIC int next_router(const smartlist_t *cons, int cur);
+STATIC int *lcs_lengths(const smartlist_slice_t *slice1,
+                        const smartlist_slice_t *slice2,
+                        int direction);
+STATIC void trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2);
+STATIC int base64cmp(const cdline_t *hash1, const cdline_t *hash2);
+STATIC int get_id_hash(const cdline_t *line, cdline_t *hash_out);
+STATIC int is_valid_router_entry(const cdline_t *line);
+STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice,
+                                      const cdline_t *string);
+STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2,
+                        const smartlist_slice_t *slice1,
+                        const smartlist_slice_t *slice2);
+STATIC int consensus_split_lines(smartlist_t *out, const char *s,
+                                 struct memarea_t *area);
+STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area,
+                                  const char *s);
+STATIC int lines_eq(const cdline_t *a, const cdline_t *b);
+STATIC int line_str_eq(const cdline_t *a, const char *b);
+
+MOCK_DECL(STATIC int,
+          consensus_compute_digest,(const char *cons,
+                                    consensus_digest_t *digest_out));
+MOCK_DECL(STATIC int,
+          consensus_compute_digest_as_signed,(const char *cons,
+                                              consensus_digest_t *digest_out));
+MOCK_DECL(STATIC int,
+          consensus_digest_eq,(const uint8_t *d1,
+                               const uint8_t *d2));
+#endif /* defined(CONSDIFF_PRIVATE) */
+
+#endif /* !defined(TOR_CONSDIFF_H) */
diff --git a/src/feature/dircommon/dir_connection_st.h b/src/feature/dircommon/dir_connection_st.h
new file mode 100644
index 0000000000..1282f82d64
--- /dev/null
+++ b/src/feature/dircommon/dir_connection_st.h
@@ -0,0 +1,67 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DIR_CONNECTION_ST_H
+#define DIR_CONNECTION_ST_H
+
+#include "or/connection_st.h"
+
+struct tor_compress_state_t;
+
+/** Subtype of connection_t for an "directory connection" -- that is, an HTTP
+ * connection to retrieve or serve directory material. */
+struct dir_connection_t {
+  connection_t base_;
+
+ /** Which 'resource' did we ask the directory for? This is typically the part
+  * of the URL string that defines, relative to the directory conn purpose,
+  * what thing we want.  For example, in router descriptor downloads by
+  * descriptor digest, it contains "d/", then one or more +-separated
+  * fingerprints.
+  **/
+  char *requested_resource;
+  unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */
+
+  /** If we're fetching descriptors, what router purpose shall we assign
+   * to them? */
+  uint8_t router_purpose;
+
+  /** List of spooled_resource_t for objects that we're spooling. We use
+   * it from back to front. */
+  smartlist_t *spool;
+  /** The compression object doing on-the-fly compression for spooled data. */
+  struct tor_compress_state_t *compress_state;
+
+  /** What rendezvous service are we querying for? */
+  rend_data_t *rend_data;
+
+  /* Hidden service connection identifier for dir connections: Used by HS
+     client-side code to fetch HS descriptors, and by the service-side code to
+     upload descriptors. */
+  struct hs_ident_dir_conn_t *hs_ident;
+
+  /** If this is a one-hop connection, tracks the state of the directory guard
+   * for this connection (if any). */
+  struct circuit_guard_state_t *guard_state;
+
+  char identity_digest[DIGEST_LEN]; /**< Hash of the public RSA key for
+                                     * the directory server's signing key. */
+
+  /** Unique ID for directory requests; this used to be in connection_t, but
+   * that's going away and being used on channels instead.  The dirserver still
+   * needs this for the incoming side, so it's moved here. */
+  uint64_t dirreq_id;
+
+#ifdef MEASUREMENTS_21206
+  /** Number of RELAY_DATA cells received. */
+  uint32_t data_cells_received;
+
+  /** Number of RELAY_DATA cells sent. */
+  uint32_t data_cells_sent;
+#endif /* defined(MEASUREMENTS_21206) */
+};
+
+#endif
diff --git a/src/feature/dircommon/fp_pair.c b/src/feature/dircommon/fp_pair.c
new file mode 100644
index 0000000000..1d7b751c23
--- /dev/null
+++ b/src/feature/dircommon/fp_pair.c
@@ -0,0 +1,315 @@
+/* Copyright (c) 2013-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fp_pair.c
+ *
+ * \brief Manages data structures for associating pairs of fingerprints. Used
+ * to handle combinations of identity/signing-key fingerprints for
+ * authorities.
+ *
+ * This is a nice, simple, compact data structure module that handles a map
+ * from (signing key fingerprint, identity key fingerprint) to void *.  The
+ * fingerprints here are SHA1 digests of RSA keys.
+ *
+ * This structure is used in directory.c and in routerlist.c for handling
+ * handling authority certificates, since we never want more than a single
+ * certificate for any (ID key, signing key) pair.
+ **/
+
+#include "or/or.h"
+#include "or/fp_pair.h"
+
+/* Define fp_pair_map_t structures */
+
+struct fp_pair_map_entry_s {
+  HT_ENTRY(fp_pair_map_entry_s) node;
+  void *val;
+  fp_pair_t key;
+};
+
+struct fp_pair_map_s {
+  HT_HEAD(fp_pair_map_impl, fp_pair_map_entry_s) head;
+};
+
+/*
+ * Hash function and equality checker for fp_pair_map_t
+ */
+
+/** Compare fp_pair_entry_t objects by key value. */
+static inline int
+fp_pair_map_entries_eq(const fp_pair_map_entry_t *a,
+                       const fp_pair_map_entry_t *b)
+{
+  return tor_memeq(&(a->key), &(b->key), sizeof(fp_pair_t));
+}
+
+/** Return a hash value for an fp_pair_entry_t. */
+static inline unsigned int
+fp_pair_map_entry_hash(const fp_pair_map_entry_t *a)
+{
+  tor_assert(sizeof(a->key) == DIGEST_LEN*2);
+  return (unsigned) siphash24g(&a->key, DIGEST_LEN*2);
+}
+
+/*
+ * Hash table functions for fp_pair_map_t
+ */
+
+HT_PROTOTYPE(fp_pair_map_impl, fp_pair_map_entry_s, node,
+             fp_pair_map_entry_hash, fp_pair_map_entries_eq)
+HT_GENERATE2(fp_pair_map_impl, fp_pair_map_entry_s, node,
+             fp_pair_map_entry_hash, fp_pair_map_entries_eq,
+             0.6, tor_reallocarray_, tor_free_)
+
+/** Constructor to create a new empty map from fp_pair_t to void *
+ */
+
+fp_pair_map_t *
+fp_pair_map_new(void)
+{
+  fp_pair_map_t *result;
+
+  result = tor_malloc(sizeof(fp_pair_map_t));
+  HT_INIT(fp_pair_map_impl, &result->head);
+  return result;
+}
+
+/** Set the current value for key to val; returns the previous
+ * value for key if one was set, or NULL if one was not.
+ */
+
+void *
+fp_pair_map_set(fp_pair_map_t *map, const fp_pair_t *key, void *val)
+{
+  fp_pair_map_entry_t *resolve;
+  fp_pair_map_entry_t search;
+  void *oldval;
+
+  tor_assert(map);
+  tor_assert(key);
+  tor_assert(val);
+
+  memcpy(&(search.key), key, sizeof(*key));
+  resolve = HT_FIND(fp_pair_map_impl, &(map->head), &search);
+  if (resolve) {
+    oldval = resolve->val;
+    resolve->val = val;
+  } else {
+    resolve = tor_malloc_zero(sizeof(fp_pair_map_entry_t));
+    memcpy(&(resolve->key), key, sizeof(*key));
+    resolve->val = val;
+    HT_INSERT(fp_pair_map_impl, &(map->head), resolve);
+    oldval = NULL;
+  }
+
+  return oldval;
+}
+
+/** Set the current value for the key (first, second) to val; returns
+ * the previous value for key if one was set, or NULL if one was not.
+ */
+
+void *
+fp_pair_map_set_by_digests(fp_pair_map_t *map,
+                           const char *first, const char *second,
+                           void *val)
+{
+  fp_pair_t k;
+
+  tor_assert(first);
+  tor_assert(second);
+
+  memcpy(k.first, first, DIGEST_LEN);
+  memcpy(k.second, second, DIGEST_LEN);
+
+  return fp_pair_map_set(map, &k, val);
+}
+
+/** Return the current value associated with key, or NULL if no value is set.
+ */
+
+void *
+fp_pair_map_get(const fp_pair_map_t *map, const fp_pair_t *key)
+{
+  fp_pair_map_entry_t *resolve;
+  fp_pair_map_entry_t search;
+  void *val = NULL;
+
+  tor_assert(map);
+  tor_assert(key);
+
+  memcpy(&(search.key), key, sizeof(*key));
+  resolve = HT_FIND(fp_pair_map_impl, &(map->head), &search);
+  if (resolve) val = resolve->val;
+
+  return val;
+}
+
+/** Return the current value associated the key (first, second), or
+ * NULL if no value is set.
+ */
+
+void *
+fp_pair_map_get_by_digests(const fp_pair_map_t *map,
+                           const char *first, const char *second)
+{
+  fp_pair_t k;
+
+  tor_assert(first);
+  tor_assert(second);
+
+  memcpy(k.first, first, DIGEST_LEN);
+  memcpy(k.second, second, DIGEST_LEN);
+
+  return fp_pair_map_get(map, &k);
+}
+
+/** Remove the value currently associated with key from the map.
+ * Return the value if one was set, or NULL if there was no entry for
+ * key.  The caller must free any storage associated with the
+ * returned value.
+ */
+
+void *
+fp_pair_map_remove(fp_pair_map_t *map, const fp_pair_t *key)
+{
+  fp_pair_map_entry_t *resolve;
+  fp_pair_map_entry_t search;
+  void *val = NULL;
+
+  tor_assert(map);
+  tor_assert(key);
+
+  memcpy(&(search.key), key, sizeof(*key));
+  resolve = HT_REMOVE(fp_pair_map_impl, &(map->head), &search);
+  if (resolve) {
+    val = resolve->val;
+    tor_free(resolve);
+  }
+
+  return val;
+}
+
+/** Remove all entries from map, and deallocate storage for those entries.
+ * If free_val is provided, it is invoked on every value in map.
+ */
+
+void
+fp_pair_map_free_(fp_pair_map_t *map, void (*free_val)(void*))
+{
+  fp_pair_map_entry_t **ent, **next, *this;
+
+  if (map) {
+    for (ent = HT_START(fp_pair_map_impl, &(map->head));
+         ent != NULL; ent = next) {
+      this = *ent;
+      next = HT_NEXT_RMV(fp_pair_map_impl, &(map->head), ent);
+      if (free_val) free_val(this->val);
+      tor_free(this);
+    }
+    tor_assert(HT_EMPTY(&(map->head)));
+    HT_CLEAR(fp_pair_map_impl, &(map->head));
+    tor_free(map);
+  }
+}
+
+/** Return true iff map has no entries.
+ */
+
+int
+fp_pair_map_isempty(const fp_pair_map_t *map)
+{
+  tor_assert(map);
+
+  return HT_EMPTY(&(map->head));
+}
+
+/** Return the number of items in map.
+ */
+
+int
+fp_pair_map_size(const fp_pair_map_t *map)
+{
+  tor_assert(map);
+
+  return HT_SIZE(&(map->head));
+}
+
+/** return an iterator pointing to the start of map.
+ */
+
+fp_pair_map_iter_t *
+fp_pair_map_iter_init(fp_pair_map_t *map)
+{
+  tor_assert(map);
+
+  return HT_START(fp_pair_map_impl, &(map->head));
+}
+
+/** Advance iter a single step to the next entry of map, and return
+ * its new value.
+ */
+
+fp_pair_map_iter_t *
+fp_pair_map_iter_next(fp_pair_map_t *map, fp_pair_map_iter_t *iter)
+{
+  tor_assert(map);
+  tor_assert(iter);
+
+  return HT_NEXT(fp_pair_map_impl, &(map->head), iter);
+}
+
+/** Advance iter a single step to the next entry of map, removing the current
+ * entry, and return its new value.
+ */
+
+fp_pair_map_iter_t *
+fp_pair_map_iter_next_rmv(fp_pair_map_t *map, fp_pair_map_iter_t *iter)
+{
+  fp_pair_map_entry_t *rmv;
+
+  tor_assert(map);
+  tor_assert(iter);
+  tor_assert(*iter);
+
+  rmv = *iter;
+  iter = HT_NEXT_RMV(fp_pair_map_impl, &(map->head), iter);
+  tor_free(rmv);
+
+  return iter;
+}
+
+/** Set *key_out and *val_out to the current entry pointed to by iter.
+ */
+
+void
+fp_pair_map_iter_get(fp_pair_map_iter_t *iter,
+                     fp_pair_t *key_out, void **val_out)
+{
+  tor_assert(iter);
+  tor_assert(*iter);
+
+  if (key_out) memcpy(key_out, &((*iter)->key), sizeof(fp_pair_t));
+  if (val_out) *val_out = (*iter)->val;
+}
+
+/** Return true iff iter has advanced past the last entry of its map.
+ */
+
+int
+fp_pair_map_iter_done(fp_pair_map_iter_t *iter)
+{
+  return (iter == NULL);
+}
+
+/** Assert if anything has gone wrong with the internal
+ * representation of map.
+ */
+
+void
+fp_pair_map_assert_ok(const fp_pair_map_t *map)
+{
+  tor_assert(!fp_pair_map_impl_HT_REP_IS_BAD_(&(map->head)));
+}
+
diff --git a/src/feature/dircommon/fp_pair.h b/src/feature/dircommon/fp_pair.h
new file mode 100644
index 0000000000..500c7c9928
--- /dev/null
+++ b/src/feature/dircommon/fp_pair.h
@@ -0,0 +1,56 @@
+/* Copyright (c) 2013-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fp_pair.h
+ * \brief Header file for fp_pair.c.
+ **/
+
+#ifndef _TOR_FP_PAIR_H
+#define _TOR_FP_PAIR_H
+
+/** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */
+typedef struct {
+  char first[DIGEST_LEN];
+  char second[DIGEST_LEN];
+} fp_pair_t;
+
+/*
+ * Declare fp_pair_map_t functions and structs
+ */
+
+typedef struct fp_pair_map_entry_s fp_pair_map_entry_t;
+typedef struct fp_pair_map_s fp_pair_map_t;
+typedef fp_pair_map_entry_t *fp_pair_map_iter_t;
+
+fp_pair_map_t * fp_pair_map_new(void);
+void * fp_pair_map_set(fp_pair_map_t *map, const fp_pair_t *key, void *val);
+void * fp_pair_map_set_by_digests(fp_pair_map_t *map,
+                                  const char *first, const char *second,
+                                  void *val);
+void * fp_pair_map_get(const fp_pair_map_t *map, const fp_pair_t *key);
+void * fp_pair_map_get_by_digests(const fp_pair_map_t *map,
+                                  const char *first, const char *second);
+void * fp_pair_map_remove(fp_pair_map_t *map, const fp_pair_t *key);
+void fp_pair_map_free_(fp_pair_map_t *map, void (*free_val)(void*));
+#define fp_pair_map_free(map, free_val) do {            \
+    fp_pair_map_free_((map), (free_val));               \
+    (map) = NULL;                                       \
+  } while (0)
+
+int fp_pair_map_isempty(const fp_pair_map_t *map);
+int fp_pair_map_size(const fp_pair_map_t *map);
+fp_pair_map_iter_t * fp_pair_map_iter_init(fp_pair_map_t *map);
+fp_pair_map_iter_t * fp_pair_map_iter_next(fp_pair_map_t *map,
+                                           fp_pair_map_iter_t *iter);
+fp_pair_map_iter_t * fp_pair_map_iter_next_rmv(fp_pair_map_t *map,
+                                               fp_pair_map_iter_t *iter);
+void fp_pair_map_iter_get(fp_pair_map_iter_t *iter,
+                          fp_pair_t *key_out, void **val_out);
+int fp_pair_map_iter_done(fp_pair_map_iter_t *iter);
+void fp_pair_map_assert_ok(const fp_pair_map_t *map);
+
+#undef DECLARE_MAP_FNS
+
+#endif /* !defined(_TOR_FP_PAIR_H) */
+
diff --git a/src/feature/dircommon/vote_timing_st.h b/src/feature/dircommon/vote_timing_st.h
new file mode 100644
index 0000000000..14c13eed28
--- /dev/null
+++ b/src/feature/dircommon/vote_timing_st.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef VOTE_TIMING_ST_H
+#define VOTE_TIMING_ST_H
+
+/** Describes the schedule by which votes should be generated. */
+struct vote_timing_t {
+  /** Length in seconds between one consensus becoming valid and the next
+   * becoming valid. */
+  int vote_interval;
+  /** For how many intervals is a consensus valid? */
+  int n_intervals_valid;
+  /** Time in seconds allowed to propagate votes */
+  int vote_delay;
+  /** Time in seconds allowed to propagate signatures */
+  int dist_delay;
+};
+
+#endif
+
diff --git a/src/feature/dircommon/voting_schedule.c b/src/feature/dircommon/voting_schedule.c
new file mode 100644
index 0000000000..6edde3f229
--- /dev/null
+++ b/src/feature/dircommon/voting_schedule.c
@@ -0,0 +1,194 @@
+/* Copyright (c) 2018-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voting_schedule.c
+ * \brief This file contains functions that are from the directory authority
+ *        subsystem related to voting specifically but used by many part of
+ *        tor. The full feature is built as part of the dirauth module.
+ **/
+
+#define VOTING_SCHEDULE_PRIVATE
+#include "or/voting_schedule.h"
+
+#include "or/or.h"
+#include "or/config.h"
+#include "or/networkstatus.h"
+
+#include "or/networkstatus_st.h"
+
+/* =====
+ * Vote scheduling
+ * ===== */
+
+/** Return the start of the next interval of size <b>interval</b> (in
+ * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always
+ * starts a fresh interval, and if the last interval of a day would be
+ * truncated to less than half its size, it is rolled into the
+ * previous interval. */
+time_t
+voting_schedule_get_start_of_next_interval(time_t now, int interval,
+                                           int offset)
+{
+  struct tm tm;
+  time_t midnight_today=0;
+  time_t midnight_tomorrow;
+  time_t next;
+
+  tor_gmtime_r(&now, &tm);
+  tm.tm_hour = 0;
+  tm.tm_min = 0;
+  tm.tm_sec = 0;
+
+  if (tor_timegm(&tm, &midnight_today) < 0) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight.");
+    // LCOV_EXCL_STOP
+  }
+  midnight_tomorrow = midnight_today + (24*60*60);
+
+  next = midnight_today + ((now-midnight_today)/interval + 1)*interval;
+
+  /* Intervals never cross midnight. */
+  if (next > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  /* If the interval would only last half as long as it's supposed to, then
+   * skip over to the next day. */
+  if (next + interval/2 > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  next += offset;
+  if (next - interval > now)
+    next -= interval;
+
+  return next;
+}
+
+/* Populate and return a new voting_schedule_t that can be used to schedule
+ * voting. The object is allocated on the heap and it's the responsibility of
+ * the caller to free it. Can't fail. */
+static voting_schedule_t *
+get_voting_schedule(const or_options_t *options, time_t now, int severity)
+{
+  int interval, vote_delay, dist_delay;
+  time_t start;
+  time_t end;
+  networkstatus_t *consensus;
+  voting_schedule_t *new_voting_schedule;
+
+  new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t));
+
+  consensus = networkstatus_get_live_consensus(now);
+
+  if (consensus) {
+    interval = (int)( consensus->fresh_until - consensus->valid_after );
+    vote_delay = consensus->vote_seconds;
+    dist_delay = consensus->dist_seconds;
+
+    /* Note down the consensus valid after, so that we detect outdated voting
+     * schedules in case of skewed clocks etc. */
+    new_voting_schedule->live_consensus_valid_after = consensus->valid_after;
+  } else {
+    interval = options->TestingV3AuthInitialVotingInterval;
+    vote_delay = options->TestingV3AuthInitialVoteDelay;
+    dist_delay = options->TestingV3AuthInitialDistDelay;
+  }
+
+  tor_assert(interval > 0);
+
+  if (vote_delay + dist_delay > interval/2)
+    vote_delay = dist_delay = interval / 4;
+
+  start = new_voting_schedule->interval_starts =
+    voting_schedule_get_start_of_next_interval(now,interval,
+                                      options->TestingV3AuthVotingStartOffset);
+  end = voting_schedule_get_start_of_next_interval(start+1, interval,
+                                      options->TestingV3AuthVotingStartOffset);
+
+  tor_assert(end > start);
+
+  new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2);
+  new_voting_schedule->voting_ends = start - dist_delay;
+  new_voting_schedule->fetch_missing_votes =
+    start - dist_delay - (vote_delay/2);
+  new_voting_schedule->voting_starts = start - dist_delay - vote_delay;
+
+  {
+    char tbuf[ISO_TIME_LEN+1];
+    format_iso_time(tbuf, new_voting_schedule->interval_starts);
+    tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: "
+            "consensus_set=%d, interval=%d",
+            tbuf, consensus?1:0, interval);
+  }
+
+  return new_voting_schedule;
+}
+
+#define voting_schedule_free(s) \
+  FREE_AND_NULL(voting_schedule_t, voting_schedule_free_, (s))
+
+/** Frees a voting_schedule_t. This should be used instead of the generic
+ * tor_free. */
+static void
+voting_schedule_free_(voting_schedule_t *voting_schedule_to_free)
+{
+  if (!voting_schedule_to_free)
+    return;
+  tor_free(voting_schedule_to_free);
+}
+
+voting_schedule_t voting_schedule;
+
+/* Using the time <b>now</b>, return the next voting valid-after time. */
+time_t
+voting_schedule_get_next_valid_after_time(void)
+{
+  time_t now = approx_time();
+  bool need_to_recalculate_voting_schedule = false;
+
+  /* This is a safe guard in order to make sure that the voting schedule
+   * static object is at least initialized. Using this function with a zeroed
+   * voting schedule can lead to bugs. */
+  if (tor_mem_is_zero((const char *) &voting_schedule,
+                      sizeof(voting_schedule))) {
+    need_to_recalculate_voting_schedule = true;
+    goto done; /* no need for next check if we have to recalculate anyway */
+  }
+
+  /* Also make sure we are not using an outdated voting schedule. If we have a
+   * newer consensus, make sure we recalculate the voting schedule. */
+  const networkstatus_t *ns = networkstatus_get_live_consensus(now);
+  if (ns && ns->valid_after != voting_schedule.live_consensus_valid_after) {
+    log_info(LD_DIR, "Voting schedule is outdated: recalculating (%d/%d)",
+             (int) ns->valid_after,
+             (int) voting_schedule.live_consensus_valid_after);
+    need_to_recalculate_voting_schedule = true;
+  }
+
+ done:
+  if (need_to_recalculate_voting_schedule) {
+    voting_schedule_recalculate_timing(get_options(), now);
+    voting_schedule.created_on_demand = 1;
+  }
+
+  return voting_schedule.interval_starts;
+}
+
+/** Set voting_schedule to hold the timing for the next vote we should be
+ * doing. All type of tor do that because HS subsystem needs the timing as
+ * well to function properly. */
+void
+voting_schedule_recalculate_timing(const or_options_t *options, time_t now)
+{
+  voting_schedule_t *new_voting_schedule;
+
+  /* get the new voting schedule */
+  new_voting_schedule = get_voting_schedule(options, now, LOG_INFO);
+  tor_assert(new_voting_schedule);
+
+  /* Fill in the global static struct now */
+  memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule));
+  voting_schedule_free(new_voting_schedule);
+}
+
diff --git a/src/feature/dircommon/voting_schedule.h b/src/feature/dircommon/voting_schedule.h
new file mode 100644
index 0000000000..0f27d36d52
--- /dev/null
+++ b/src/feature/dircommon/voting_schedule.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2018-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voting_schedule.h
+ * \brief Header file for voting_schedule.c.
+ **/
+
+#ifndef TOR_VOTING_SCHEDULE_H
+#define TOR_VOTING_SCHEDULE_H
+
+#include "or/or.h"
+
+/** Scheduling information for a voting interval. */
+typedef struct {
+  /** When do we generate and distribute our vote for this interval? */
+  time_t voting_starts;
+  /** When do we send an HTTP request for any votes that we haven't
+   * been posted yet?*/
+  time_t fetch_missing_votes;
+  /** When do we give up on getting more votes and generate a consensus? */
+  time_t voting_ends;
+  /** When do we send an HTTP request for any signatures we're expecting to
+   * see on the consensus? */
+  time_t fetch_missing_signatures;
+  /** When do we publish the consensus? */
+  time_t interval_starts;
+
+  /* True iff we have generated and distributed our vote. */
+  int have_voted;
+  /* True iff we've requested missing votes. */
+  int have_fetched_missing_votes;
+  /* True iff we have built a consensus and sent the signatures around. */
+  int have_built_consensus;
+  /* True iff we've fetched missing signatures. */
+  int have_fetched_missing_signatures;
+  /* True iff we have published our consensus. */
+  int have_published_consensus;
+
+  /* True iff this voting schedule was set on demand meaning not through the
+   * normal vote operation of a dirauth or when a consensus is set. This only
+   * applies to a directory authority that needs to recalculate the voting
+   * timings only for the first vote even though this object was initilized
+   * prior to voting. */
+  int created_on_demand;
+
+  /** The valid-after time of the last live consensus that filled this voting
+   *  schedule.  It's used to detect outdated voting schedules. */
+  time_t live_consensus_valid_after;
+} voting_schedule_t;
+
+/* Public API. */
+
+extern voting_schedule_t voting_schedule;
+
+void voting_schedule_recalculate_timing(const or_options_t *options,
+                                        time_t now);
+
+time_t voting_schedule_get_start_of_next_interval(time_t now,
+                                                  int interval,
+                                                  int offset);
+time_t voting_schedule_get_next_valid_after_time(void);
+
+#endif /* TOR_VOTING_SCHEDULE_H */
+
diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c
new file mode 100644
index 0000000000..55de64c13e
--- /dev/null
+++ b/src/feature/hibernate/hibernate.c
@@ -0,0 +1,1235 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hibernate.c
+ * \brief Functions to close listeners, stop allowing new circuits,
+ * etc in preparation for closing down or going dormant; and to track
+ * bandwidth and time intervals to know when to hibernate and when to
+ * stop hibernating.
+ *
+ * Ordinarily a Tor relay is "Live".
+ *
+ * A live relay can stop accepting connections for one of two reasons: either
+ * it is trying to conserve bandwidth because of bandwidth accounting rules
+ * ("soft hibernation"), or it is about to shut down ("exiting").
+ **/
+
+/*
+hibernating, phase 1:
+  - send destroy in response to create cells
+  - send end (policy failed) in response to begin cells
+  - close an OR conn when it has no circuits
+
+hibernating, phase 2:
+  (entered when bandwidth hard limit reached)
+  - close all OR/AP/exit conns)
+*/
+
+#define HIBERNATE_PRIVATE
+#include "or/or.h"
+#include "or/channel.h"
+#include "or/channeltls.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/connection_edge.h"
+#include "or/connection_or.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "or/hibernate.h"
+#include "or/main.h"
+#include "or/router.h"
+#include "or/statefile.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include "or/or_connection_st.h"
+#include "or/or_state_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/** Are we currently awake, asleep, running out of bandwidth, or shutting
+ * down? */
+static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL;
+/** If are hibernating, when do we plan to wake up? Set to 0 if we
+ * aren't hibernating. */
+static time_t hibernate_end_time = 0;
+/** If we are shutting down, when do we plan finally exit? Set to 0 if
+ * we aren't shutting down. */
+static time_t shutdown_time = 0;
+
+/** A timed event that we'll use when it's time to wake up from
+ * hibernation. */
+static mainloop_event_t *wakeup_event = NULL;
+
+/** Possible accounting periods. */
+typedef enum {
+  UNIT_MONTH=1, UNIT_WEEK=2, UNIT_DAY=3,
+} time_unit_t;
+
+/*
+ * @file hibernate.c
+ *
+ * <h4>Accounting</h4>
+ * Accounting is designed to ensure that no more than N bytes are sent in
+ * either direction over a given interval (currently, one month, one week, or
+ * one day) We could
+ * try to do this by choking our bandwidth to a trickle, but that
+ * would make our streams useless.  Instead, we estimate what our
+ * bandwidth usage will be, and guess how long we'll be able to
+ * provide that much bandwidth before hitting our limit.  We then
+ * choose a random time within the accounting interval to come up (so
+ * that we don't get 50 Tors running on the 1st of the month and none
+ * on the 30th).
+ *
+ * Each interval runs as follows:
+ *
+ * <ol>
+ * <li>We guess our bandwidth usage, based on how much we used
+ *     last time.  We choose a "wakeup time" within the interval to come up.
+ * <li>Until the chosen wakeup time, we hibernate.
+ * <li> We come up at the wakeup time, and provide bandwidth until we are
+ *    "very close" to running out.
+ * <li> Then we go into low-bandwidth mode, and stop accepting new
+ *    connections, but provide bandwidth until we run out.
+ * <li> Then we hibernate until the end of the interval.
+ *
+ * If the interval ends before we run out of bandwidth, we go back to
+ * step one.
+ *
+ * Accounting is controlled by the AccountingMax, AccountingRule, and
+ * AccountingStart options.
+ */
+
+/** How many bytes have we read in this accounting interval? */
+static uint64_t n_bytes_read_in_interval = 0;
+/** How many bytes have we written in this accounting interval? */
+static uint64_t n_bytes_written_in_interval = 0;
+/** How many seconds have we been running this interval? */
+static uint32_t n_seconds_active_in_interval = 0;
+/** How many seconds were we active in this interval before we hit our soft
+ * limit? */
+static int n_seconds_to_hit_soft_limit = 0;
+/** When in this interval was the soft limit hit. */
+static time_t soft_limit_hit_at = 0;
+/** How many bytes had we read/written when we hit the soft limit? */
+static uint64_t n_bytes_at_soft_limit = 0;
+/** When did this accounting interval start? */
+static time_t interval_start_time = 0;
+/** When will this accounting interval end? */
+static time_t interval_end_time = 0;
+/** How far into the accounting interval should we hibernate? */
+static time_t interval_wakeup_time = 0;
+/** How much bandwidth do we 'expect' to use per minute?  (0 if we have no
+ * info from the last period.) */
+static uint64_t expected_bandwidth_usage = 0;
+/** What unit are we using for our accounting? */
+static time_unit_t cfg_unit = UNIT_MONTH;
+
+/** How many days,hours,minutes into each unit does our accounting interval
+ * start? */
+/** @{ */
+static int cfg_start_day = 0,
+           cfg_start_hour = 0,
+           cfg_start_min = 0;
+/** @} */
+
+static const char *hibernate_state_to_string(hibernate_state_t state);
+static void reset_accounting(time_t now);
+static int read_bandwidth_usage(void);
+static time_t start_of_accounting_period_after(time_t now);
+static time_t start_of_accounting_period_containing(time_t now);
+static void accounting_set_wakeup_time(void);
+static void on_hibernate_state_change(hibernate_state_t prev_state);
+static void hibernate_schedule_wakeup_event(time_t now, time_t end_time);
+static void wakeup_event_callback(mainloop_event_t *ev, void *data);
+
+/**
+ * Return the human-readable name for the hibernation state <b>state</b>
+ */
+static const char *
+hibernate_state_to_string(hibernate_state_t state)
+{
+  static char buf[64];
+  switch (state) {
+    case HIBERNATE_STATE_EXITING: return "EXITING";
+    case HIBERNATE_STATE_LOWBANDWIDTH: return "SOFT";
+    case HIBERNATE_STATE_DORMANT: return "HARD";
+    case HIBERNATE_STATE_INITIAL:
+    case HIBERNATE_STATE_LIVE:
+      return "AWAKE";
+    default:
+      log_warn(LD_BUG, "unknown hibernate state %d", state);
+      tor_snprintf(buf, sizeof(buf), "unknown [%d]", state);
+      return buf;
+  }
+}
+
+/* ************
+ * Functions for bandwidth accounting.
+ * ************/
+
+/** Configure accounting start/end time settings based on
+ * options->AccountingStart.  Return 0 on success, -1 on failure. If
+ * <b>validate_only</b> is true, do not change the current settings. */
+int
+accounting_parse_options(const or_options_t *options, int validate_only)
+{
+  time_unit_t unit;
+  int ok, idx;
+  long d,h,m;
+  smartlist_t *items;
+  const char *v = options->AccountingStart;
+  const char *s;
+  char *cp;
+
+  if (!v) {
+    if (!validate_only) {
+      cfg_unit = UNIT_MONTH;
+      cfg_start_day = 1;
+      cfg_start_hour = 0;
+      cfg_start_min = 0;
+    }
+    return 0;
+  }
+
+  items = smartlist_new();
+  smartlist_split_string(items, v, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK,0);
+  if (smartlist_len(items)<2) {
+    log_warn(LD_CONFIG, "Too few arguments to AccountingStart");
+    goto err;
+  }
+  s = smartlist_get(items,0);
+  if (0==strcasecmp(s, "month")) {
+    unit = UNIT_MONTH;
+  } else if (0==strcasecmp(s, "week")) {
+    unit = UNIT_WEEK;
+  } else if (0==strcasecmp(s, "day")) {
+    unit = UNIT_DAY;
+  } else {
+    log_warn(LD_CONFIG,
+             "Unrecognized accounting unit '%s': only 'month', 'week',"
+             " and 'day' are supported.", s);
+    goto err;
+  }
+
+  switch (unit) {
+  case UNIT_WEEK:
+    d = tor_parse_long(smartlist_get(items,1), 10, 1, 7, &ok, NULL);
+    if (!ok) {
+      log_warn(LD_CONFIG, "Weekly accounting must begin on a day between "
+               "1 (Monday) and 7 (Sunday)");
+      goto err;
+    }
+    break;
+  case UNIT_MONTH:
+    d = tor_parse_long(smartlist_get(items,1), 10, 1, 28, &ok, NULL);
+    if (!ok) {
+      log_warn(LD_CONFIG, "Monthly accounting must begin on a day between "
+               "1 and 28");
+      goto err;
+    }
+    break;
+  case UNIT_DAY:
+    d = 0;
+    break;
+    /* Coverity dislikes unreachable default cases; some compilers warn on
+     * switch statements missing a case.  Tell Coverity not to worry. */
+    /* coverity[dead_error_begin] */
+  default:
+    tor_assert(0);
+  }
+
+  idx = unit==UNIT_DAY?1:2;
+  if (smartlist_len(items) != (idx+1)) {
+    log_warn(LD_CONFIG,"Accounting unit '%s' requires %d argument%s.",
+             s, idx, (idx>1)?"s":"");
+    goto err;
+  }
+  s = smartlist_get(items, idx);
+  h = tor_parse_long(s, 10, 0, 23, &ok, &cp);
+  if (!ok) {
+    log_warn(LD_CONFIG,"Accounting start time not parseable: bad hour.");
+    goto err;
+  }
+  if (!cp || *cp!=':') {
+    log_warn(LD_CONFIG,
+             "Accounting start time not parseable: not in HH:MM format");
+    goto err;
+  }
+  m = tor_parse_long(cp+1, 10, 0, 59, &ok, &cp);
+  if (!ok) {
+    log_warn(LD_CONFIG, "Accounting start time not parseable: bad minute");
+    goto err;
+  }
+  if (!cp || *cp!='\0') {
+    log_warn(LD_CONFIG,
+             "Accounting start time not parseable: not in HH:MM format");
+    goto err;
+  }
+
+  if (!validate_only) {
+    cfg_unit = unit;
+    cfg_start_day = (int)d;
+    cfg_start_hour = (int)h;
+    cfg_start_min = (int)m;
+  }
+  SMARTLIST_FOREACH(items, char *, item, tor_free(item));
+  smartlist_free(items);
+  return 0;
+ err:
+  SMARTLIST_FOREACH(items, char *, item, tor_free(item));
+  smartlist_free(items);
+  return -1;
+}
+
+/** If we want to manage the accounting system and potentially
+ * hibernate, return 1, else return 0.
+ */
+MOCK_IMPL(int,
+accounting_is_enabled,(const or_options_t *options))
+{
+  if (options->AccountingMax)
+    return 1;
+  return 0;
+}
+
+/** If accounting is enabled, return how long (in seconds) this
+ * interval lasts. */
+int
+accounting_get_interval_length(void)
+{
+  return (int)(interval_end_time - interval_start_time);
+}
+
+/** Return the time at which the current accounting interval will end. */
+MOCK_IMPL(time_t,
+accounting_get_end_time,(void))
+{
+  return interval_end_time;
+}
+
+/** Called from connection.c to tell us that <b>seconds</b> seconds have
+ * passed, <b>n_read</b> bytes have been read, and <b>n_written</b>
+ * bytes have been written. */
+void
+accounting_add_bytes(size_t n_read, size_t n_written, int seconds)
+{
+  n_bytes_read_in_interval += n_read;
+  n_bytes_written_in_interval += n_written;
+  /* If we haven't been called in 10 seconds, we're probably jumping
+   * around in time. */
+  n_seconds_active_in_interval += (seconds < 10) ? seconds : 0;
+}
+
+/** If get_end, return the end of the accounting period that contains
+ * the time <b>now</b>.  Else, return the start of the accounting
+ * period that contains the time <b>now</b> */
+static time_t
+edge_of_accounting_period_containing(time_t now, int get_end)
+{
+  int before;
+  struct tm tm;
+  tor_localtime_r(&now, &tm);
+
+  /* Set 'before' to true iff the current time is before the hh:mm
+   * changeover time for today. */
+  before = tm.tm_hour < cfg_start_hour ||
+    (tm.tm_hour == cfg_start_hour && tm.tm_min < cfg_start_min);
+
+  /* Dispatch by unit.  First, find the start day of the given period;
+   * then, if get_end is true, increment to the end day. */
+  switch (cfg_unit)
+    {
+    case UNIT_MONTH: {
+      /* If this is before the Nth, we want the Nth of last month. */
+      if (tm.tm_mday < cfg_start_day ||
+          (tm.tm_mday == cfg_start_day && before)) {
+        --tm.tm_mon;
+      }
+      /* Otherwise, the month is correct. */
+      tm.tm_mday = cfg_start_day;
+      if (get_end)
+        ++tm.tm_mon;
+      break;
+    }
+    case UNIT_WEEK: {
+      /* What is the 'target' day of the week in struct tm format? (We
+         say Sunday==7; struct tm says Sunday==0.) */
+      int wday = cfg_start_day % 7;
+      /* How many days do we subtract from today to get to the right day? */
+      int delta = (7+tm.tm_wday-wday)%7;
+      /* If we are on the right day, but the changeover hasn't happened yet,
+       * then subtract a whole week. */
+      if (delta == 0 && before)
+        delta = 7;
+      tm.tm_mday -= delta;
+      if (get_end)
+        tm.tm_mday += 7;
+      break;
+    }
+    case UNIT_DAY:
+      if (before)
+        --tm.tm_mday;
+      if (get_end)
+        ++tm.tm_mday;
+      break;
+    default:
+      tor_assert(0);
+  }
+
+  tm.tm_hour = cfg_start_hour;
+  tm.tm_min = cfg_start_min;
+  tm.tm_sec = 0;
+  tm.tm_isdst = -1; /* Autodetect DST */
+  return mktime(&tm);
+}
+
+/** Return the start of the accounting period containing the time
+ * <b>now</b>. */
+static time_t
+start_of_accounting_period_containing(time_t now)
+{
+  return edge_of_accounting_period_containing(now, 0);
+}
+
+/** Return the start of the accounting period that comes after the one
+ * containing the time <b>now</b>. */
+static time_t
+start_of_accounting_period_after(time_t now)
+{
+  return edge_of_accounting_period_containing(now, 1);
+}
+
+/** Return the length of the accounting period containing the time
+ * <b>now</b>. */
+static long
+length_of_accounting_period_containing(time_t now)
+{
+  return edge_of_accounting_period_containing(now, 1) -
+    edge_of_accounting_period_containing(now, 0);
+}
+
+/** Initialize the accounting subsystem. */
+void
+configure_accounting(time_t now)
+{
+  time_t s_now;
+  /* Try to remember our recorded usage. */
+  if (!interval_start_time)
+    read_bandwidth_usage(); /* If we fail, we'll leave values at zero, and
+                             * reset below.*/
+
+  s_now = start_of_accounting_period_containing(now);
+
+  if (!interval_start_time) {
+    /* We didn't have recorded usage; Start a new interval. */
+    log_info(LD_ACCT, "Starting new accounting interval.");
+    reset_accounting(now);
+  } else if (s_now == interval_start_time) {
+    log_info(LD_ACCT, "Continuing accounting interval.");
+    /* We are in the interval we thought we were in. Do nothing.*/
+    interval_end_time = start_of_accounting_period_after(interval_start_time);
+  } else {
+    long duration =
+      length_of_accounting_period_containing(interval_start_time);
+    double delta = ((double)(s_now - interval_start_time)) / duration;
+    if (-0.50 <= delta && delta <= 0.50) {
+      /* The start of the period is now a little later or earlier than we
+       * remembered.  That's fine; we might lose some bytes we could otherwise
+       * have written, but better to err on the side of obeying accounting
+       * settings. */
+      log_info(LD_ACCT, "Accounting interval moved by %.02f%%; "
+               "that's fine.", delta*100);
+      interval_end_time = start_of_accounting_period_after(now);
+    } else if (delta >= 0.99) {
+      /* This is the regular time-moved-forward case; don't be too noisy
+       * about it or people will complain */
+      log_info(LD_ACCT, "Accounting interval elapsed; starting a new one");
+      reset_accounting(now);
+    } else {
+      log_warn(LD_ACCT,
+               "Mismatched accounting interval: moved by %.02f%%. "
+               "Starting a fresh one.", delta*100);
+      reset_accounting(now);
+    }
+  }
+  accounting_set_wakeup_time();
+}
+
+/** Return the relevant number of bytes sent/received this interval
+ * based on the set AccountingRule */
+uint64_t
+get_accounting_bytes(void)
+{
+  if (get_options()->AccountingRule == ACCT_SUM)
+    return n_bytes_read_in_interval+n_bytes_written_in_interval;
+  else if (get_options()->AccountingRule == ACCT_IN)
+    return n_bytes_read_in_interval;
+  else if (get_options()->AccountingRule == ACCT_OUT)
+    return n_bytes_written_in_interval;
+  else
+    return MAX(n_bytes_read_in_interval, n_bytes_written_in_interval);
+}
+
+/** Set expected_bandwidth_usage based on how much we sent/received
+ * per minute last interval (if we were up for at least 30 minutes),
+ * or based on our declared bandwidth otherwise. */
+static void
+update_expected_bandwidth(void)
+{
+  uint64_t expected;
+  const or_options_t *options= get_options();
+  uint64_t max_configured = (options->RelayBandwidthRate > 0 ?
+                             options->RelayBandwidthRate :
+                             options->BandwidthRate) * 60;
+  /* max_configured is the larger of bytes read and bytes written
+   * If we are accounting based on sum, worst case is both are
+   * at max, doubling the expected sum of bandwidth */
+  if (get_options()->AccountingRule == ACCT_SUM)
+    max_configured *= 2;
+
+#define MIN_TIME_FOR_MEASUREMENT (1800)
+
+  if (soft_limit_hit_at > interval_start_time && n_bytes_at_soft_limit &&
+      (soft_limit_hit_at - interval_start_time) > MIN_TIME_FOR_MEASUREMENT) {
+    /* If we hit our soft limit last time, only count the bytes up to that
+     * time. This is a better predictor of our actual bandwidth than
+     * considering the entirety of the last interval, since we likely started
+     * using bytes very slowly once we hit our soft limit. */
+    expected = n_bytes_at_soft_limit /
+      (soft_limit_hit_at - interval_start_time);
+    expected /= 60;
+  } else if (n_seconds_active_in_interval >= MIN_TIME_FOR_MEASUREMENT) {
+    /* Otherwise, we either measured enough time in the last interval but
+     * never hit our soft limit, or we're using a state file from a Tor that
+     * doesn't know to store soft-limit info.  Just take rate at which
+     * we were reading/writing in the last interval as our expected rate.
+     */
+    uint64_t used = get_accounting_bytes();
+    expected = used / (n_seconds_active_in_interval / 60);
+  } else {
+    /* If we haven't gotten enough data last interval, set 'expected'
+     * to 0.  This will set our wakeup to the start of the interval.
+     * Next interval, we'll choose our starting time based on how much
+     * we sent this interval.
+     */
+    expected = 0;
+  }
+  if (expected > max_configured)
+    expected = max_configured;
+  expected_bandwidth_usage = expected;
+}
+
+/** Called at the start of a new accounting interval: reset our
+ * expected bandwidth usage based on what happened last time, set up
+ * the start and end of the interval, and clear byte/time totals.
+ */
+static void
+reset_accounting(time_t now)
+{
+  log_info(LD_ACCT, "Starting new accounting interval.");
+  update_expected_bandwidth();
+  interval_start_time = start_of_accounting_period_containing(now);
+  interval_end_time = start_of_accounting_period_after(interval_start_time);
+  n_bytes_read_in_interval = 0;
+  n_bytes_written_in_interval = 0;
+  n_seconds_active_in_interval = 0;
+  n_bytes_at_soft_limit = 0;
+  soft_limit_hit_at = 0;
+  n_seconds_to_hit_soft_limit = 0;
+}
+
+/** Return true iff we should save our bandwidth usage to disk. */
+static inline int
+time_to_record_bandwidth_usage(time_t now)
+{
+  /* Note every 600 sec */
+#define NOTE_INTERVAL (600)
+  /* Or every 20 megabytes */
+#define NOTE_BYTES 20*(1024*1024)
+  static uint64_t last_read_bytes_noted = 0;
+  static uint64_t last_written_bytes_noted = 0;
+  static time_t last_time_noted = 0;
+
+  if (last_time_noted + NOTE_INTERVAL <= now ||
+      last_read_bytes_noted + NOTE_BYTES <= n_bytes_read_in_interval ||
+      last_written_bytes_noted + NOTE_BYTES <= n_bytes_written_in_interval ||
+      (interval_end_time && interval_end_time <= now)) {
+    last_time_noted = now;
+    last_read_bytes_noted = n_bytes_read_in_interval;
+    last_written_bytes_noted = n_bytes_written_in_interval;
+    return 1;
+  }
+  return 0;
+}
+
+/** Invoked once per second.  Checks whether it is time to hibernate,
+ * record bandwidth used, etc.  */
+void
+accounting_run_housekeeping(time_t now)
+{
+  if (now >= interval_end_time) {
+    configure_accounting(now);
+  }
+  if (time_to_record_bandwidth_usage(now)) {
+    if (accounting_record_bandwidth_usage(now, get_or_state())) {
+      log_warn(LD_FS, "Couldn't record bandwidth usage to disk.");
+    }
+  }
+}
+
+/** Based on our interval and our estimated bandwidth, choose a
+ * deterministic (but random-ish) time to wake up. */
+static void
+accounting_set_wakeup_time(void)
+{
+  char digest[DIGEST_LEN];
+  crypto_digest_t *d_env;
+  uint64_t time_to_exhaust_bw;
+  int time_to_consider;
+
+  if (! server_identity_key_is_set()) {
+    if (init_keys() < 0) {
+      log_err(LD_BUG, "Error initializing keys");
+      tor_assert(0);
+    }
+  }
+
+  if (server_identity_key_is_set()) {
+    char buf[ISO_TIME_LEN+1];
+    format_iso_time(buf, interval_start_time);
+
+    if (crypto_pk_get_digest(get_server_identity_key(), digest) < 0) {
+      log_err(LD_BUG, "Error getting our key's digest.");
+      tor_assert(0);
+    }
+
+    d_env = crypto_digest_new();
+    crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN);
+    crypto_digest_add_bytes(d_env, digest, DIGEST_LEN);
+    crypto_digest_get_digest(d_env, digest, DIGEST_LEN);
+    crypto_digest_free(d_env);
+  } else {
+    crypto_rand(digest, DIGEST_LEN);
+  }
+
+  if (!expected_bandwidth_usage) {
+    char buf1[ISO_TIME_LEN+1];
+    char buf2[ISO_TIME_LEN+1];
+    format_local_iso_time(buf1, interval_start_time);
+    format_local_iso_time(buf2, interval_end_time);
+    interval_wakeup_time = interval_start_time;
+
+    log_notice(LD_ACCT,
+           "Configured hibernation. This interval begins at %s "
+           "and ends at %s. We have no prior estimate for bandwidth, so "
+           "we will start out awake and hibernate when we exhaust our quota.",
+           buf1, buf2);
+    return;
+  }
+
+  time_to_exhaust_bw =
+    (get_options()->AccountingMax/expected_bandwidth_usage)*60;
+  if (time_to_exhaust_bw > INT_MAX) {
+    time_to_exhaust_bw = INT_MAX;
+    time_to_consider = 0;
+  } else {
+    time_to_consider = accounting_get_interval_length() -
+                       (int)time_to_exhaust_bw;
+  }
+
+  if (time_to_consider<=0) {
+    interval_wakeup_time = interval_start_time;
+  } else {
+    /* XXX can we simplify this just by picking a random (non-deterministic)
+     * time to be up? If we go down and come up, then we pick a new one. Is
+     * that good enough? -RD */
+
+    /* This is not a perfectly unbiased conversion, but it is good enough:
+     * in the worst case, the first half of the day is 0.06 percent likelier
+     * to be chosen than the last half. */
+    interval_wakeup_time = interval_start_time +
+      (get_uint32(digest) % time_to_consider);
+  }
+
+  {
+    char buf1[ISO_TIME_LEN+1];
+    char buf2[ISO_TIME_LEN+1];
+    char buf3[ISO_TIME_LEN+1];
+    char buf4[ISO_TIME_LEN+1];
+    time_t down_time;
+    if (interval_wakeup_time+time_to_exhaust_bw > TIME_MAX)
+      down_time = TIME_MAX;
+    else
+      down_time = (time_t)(interval_wakeup_time+time_to_exhaust_bw);
+    if (down_time>interval_end_time)
+      down_time = interval_end_time;
+    format_local_iso_time(buf1, interval_start_time);
+    format_local_iso_time(buf2, interval_wakeup_time);
+    format_local_iso_time(buf3, down_time);
+    format_local_iso_time(buf4, interval_end_time);
+
+    log_notice(LD_ACCT,
+           "Configured hibernation.  This interval began at %s; "
+           "the scheduled wake-up time %s %s; "
+           "we expect%s to exhaust our quota for this interval around %s; "
+           "the next interval begins at %s (all times local)",
+           buf1,
+           time(NULL)<interval_wakeup_time?"is":"was", buf2,
+           time(NULL)<down_time?"":"ed", buf3,
+           buf4);
+  }
+}
+
+/* This rounds 0 up to 1000, but that's actually a feature. */
+#define ROUND_UP(x) (((x) + 0x3ff) & ~0x3ff)
+/** Save all our bandwidth tracking information to disk. Return 0 on
+ * success, -1 on failure. */
+int
+accounting_record_bandwidth_usage(time_t now, or_state_t *state)
+{
+  /* Just update the state */
+  state->AccountingIntervalStart = interval_start_time;
+  state->AccountingBytesReadInInterval = ROUND_UP(n_bytes_read_in_interval);
+  state->AccountingBytesWrittenInInterval =
+    ROUND_UP(n_bytes_written_in_interval);
+  state->AccountingSecondsActive = n_seconds_active_in_interval;
+  state->AccountingExpectedUsage = expected_bandwidth_usage;
+
+  state->AccountingSecondsToReachSoftLimit = n_seconds_to_hit_soft_limit;
+  state->AccountingSoftLimitHitAt = soft_limit_hit_at;
+  state->AccountingBytesAtSoftLimit = n_bytes_at_soft_limit;
+
+  or_state_mark_dirty(state,
+                      now+(get_options()->AvoidDiskWrites ? 7200 : 60));
+
+  return 0;
+}
+#undef ROUND_UP
+
+/** Read stored accounting information from disk. Return 0 on success;
+ * return -1 and change nothing on failure. */
+static int
+read_bandwidth_usage(void)
+{
+  or_state_t *state = get_or_state();
+
+  {
+    char *fname = get_datadir_fname("bw_accounting");
+    int res;
+
+    res = unlink(fname);
+    if (res != 0 && errno != ENOENT) {
+      log_warn(LD_FS,
+               "Failed to unlink %s: %s",
+               fname, strerror(errno));
+    }
+
+    tor_free(fname);
+  }
+
+  if (!state)
+    return -1;
+
+  log_info(LD_ACCT, "Reading bandwidth accounting data from state file");
+  n_bytes_read_in_interval = state->AccountingBytesReadInInterval;
+  n_bytes_written_in_interval = state->AccountingBytesWrittenInInterval;
+  n_seconds_active_in_interval = state->AccountingSecondsActive;
+  interval_start_time = state->AccountingIntervalStart;
+  expected_bandwidth_usage = state->AccountingExpectedUsage;
+
+  /* Older versions of Tor (before 0.2.2.17-alpha or so) didn't generate these
+   * fields. If you switch back and forth, you might get an
+   * AccountingSoftLimitHitAt value from long before the most recent
+   * interval_start_time.  If that's so, then ignore the softlimit-related
+   * values. */
+  if (state->AccountingSoftLimitHitAt > interval_start_time) {
+    soft_limit_hit_at =  state->AccountingSoftLimitHitAt;
+    n_bytes_at_soft_limit = state->AccountingBytesAtSoftLimit;
+    n_seconds_to_hit_soft_limit = state->AccountingSecondsToReachSoftLimit;
+  } else {
+    soft_limit_hit_at = 0;
+    n_bytes_at_soft_limit = 0;
+    n_seconds_to_hit_soft_limit = 0;
+  }
+
+  {
+    char tbuf1[ISO_TIME_LEN+1];
+    char tbuf2[ISO_TIME_LEN+1];
+    format_iso_time(tbuf1, state->LastWritten);
+    format_iso_time(tbuf2, state->AccountingIntervalStart);
+
+    log_info(LD_ACCT,
+       "Successfully read bandwidth accounting info from state written at %s "
+       "for interval starting at %s.  We have been active for %lu seconds in "
+       "this interval.  At the start of the interval, we expected to use "
+       "about %lu KB per second. (%"PRIu64" bytes read so far, "
+       "%"PRIu64" bytes written so far)",
+       tbuf1, tbuf2,
+       (unsigned long)n_seconds_active_in_interval,
+       (unsigned long)(expected_bandwidth_usage*1024/60),
+       (n_bytes_read_in_interval),
+       (n_bytes_written_in_interval));
+  }
+
+  return 0;
+}
+
+/** Return true iff we have sent/received all the bytes we are willing
+ * to send/receive this interval. */
+static int
+hibernate_hard_limit_reached(void)
+{
+  uint64_t hard_limit = get_options()->AccountingMax;
+  if (!hard_limit)
+    return 0;
+  return get_accounting_bytes() >= hard_limit;
+}
+
+/** Return true iff we have sent/received almost all the bytes we are willing
+ * to send/receive this interval. */
+static int
+hibernate_soft_limit_reached(void)
+{
+  const uint64_t acct_max = get_options()->AccountingMax;
+#define SOFT_LIM_PCT (.95)
+#define SOFT_LIM_BYTES (500*1024*1024)
+#define SOFT_LIM_MINUTES (3*60)
+  /* The 'soft limit' is a fair bit more complicated now than once it was.
+   * We want to stop accepting connections when ALL of the following are true:
+   *   - We expect to use up the remaining bytes in under 3 hours
+   *   - We have used up 95% of our bytes.
+   *   - We have less than 500MB of bytes left.
+   */
+  uint64_t soft_limit = (uint64_t) (acct_max * SOFT_LIM_PCT);
+  if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) {
+    soft_limit = acct_max - SOFT_LIM_BYTES;
+  }
+  if (expected_bandwidth_usage) {
+    const uint64_t expected_usage =
+      expected_bandwidth_usage * SOFT_LIM_MINUTES;
+    if (acct_max > expected_usage && acct_max - expected_usage > soft_limit)
+      soft_limit = acct_max - expected_usage;
+  }
+
+  if (!soft_limit)
+    return 0;
+  return get_accounting_bytes() >= soft_limit;
+}
+
+/** Called when we get a SIGINT, or when bandwidth soft limit is
+ * reached. Puts us into "loose hibernation": we don't accept new
+ * connections, but we continue handling old ones. */
+static void
+hibernate_begin(hibernate_state_t new_state, time_t now)
+{
+  const or_options_t *options = get_options();
+
+  if (new_state == HIBERNATE_STATE_EXITING &&
+      hibernate_state != HIBERNATE_STATE_LIVE) {
+    log_notice(LD_GENERAL,"SIGINT received %s; exiting now.",
+               hibernate_state == HIBERNATE_STATE_EXITING ?
+               "a second time" : "while hibernating");
+    tor_shutdown_event_loop_and_exit(0);
+    return;
+  }
+
+  if (new_state == HIBERNATE_STATE_LOWBANDWIDTH &&
+      hibernate_state == HIBERNATE_STATE_LIVE) {
+    soft_limit_hit_at = now;
+    n_seconds_to_hit_soft_limit = n_seconds_active_in_interval;
+    n_bytes_at_soft_limit = get_accounting_bytes();
+  }
+
+  /* close listeners. leave control listener(s). */
+  connection_mark_all_noncontrol_listeners();
+
+  /* XXX kill intro point circs */
+  /* XXX upload rendezvous service descriptors with no intro points */
+
+  if (new_state == HIBERNATE_STATE_EXITING) {
+    log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new "
+               "connections, and will shut down in %d seconds. Interrupt "
+               "again to exit now.", options->ShutdownWaitLength);
+    shutdown_time = time(NULL) + options->ShutdownWaitLength;
+  } else { /* soft limit reached */
+    hibernate_end_time = interval_end_time;
+  }
+
+  hibernate_state = new_state;
+  accounting_record_bandwidth_usage(now, get_or_state());
+
+  or_state_mark_dirty(get_or_state(),
+                      get_options()->AvoidDiskWrites ? now+600 : 0);
+}
+
+/** Called when we've been hibernating and our timeout is reached. */
+static void
+hibernate_end(hibernate_state_t new_state)
+{
+  tor_assert(hibernate_state == HIBERNATE_STATE_LOWBANDWIDTH ||
+             hibernate_state == HIBERNATE_STATE_DORMANT ||
+             hibernate_state == HIBERNATE_STATE_INITIAL);
+
+  /* listeners will be relaunched in run_scheduled_events() in main.c */
+  if (hibernate_state != HIBERNATE_STATE_INITIAL)
+    log_notice(LD_ACCT,"Hibernation period ended. Resuming normal activity.");
+
+  hibernate_state = new_state;
+  hibernate_end_time = 0; /* no longer hibernating */
+  reset_uptime(); /* reset published uptime */
+}
+
+/** A wrapper around hibernate_begin, for when we get SIGINT. */
+void
+hibernate_begin_shutdown(void)
+{
+  hibernate_begin(HIBERNATE_STATE_EXITING, time(NULL));
+}
+
+/**
+ * Return true iff we are currently hibernating -- that is, if we are in
+ * any non-live state.
+ */
+MOCK_IMPL(int,
+we_are_hibernating,(void))
+{
+  return hibernate_state != HIBERNATE_STATE_LIVE;
+}
+
+/**
+ * Return true iff we are currently _fully_ hibernating -- that is, if we are
+ * in a state where we expect to handle no network activity at all.
+ */
+MOCK_IMPL(int,
+we_are_fully_hibernating,(void))
+{
+  return hibernate_state == HIBERNATE_STATE_DORMANT;
+}
+
+/** If we aren't currently dormant, close all connections and become
+ * dormant. */
+static void
+hibernate_go_dormant(time_t now)
+{
+  connection_t *conn;
+
+  if (hibernate_state == HIBERNATE_STATE_DORMANT)
+    return;
+  else if (hibernate_state == HIBERNATE_STATE_LOWBANDWIDTH)
+    hibernate_state = HIBERNATE_STATE_DORMANT;
+  else
+    hibernate_begin(HIBERNATE_STATE_DORMANT, now);
+
+  log_notice(LD_ACCT,"Going dormant. Blowing away remaining connections.");
+
+  /* Close all OR/AP/exit conns. Leave dir conns because we still want
+   * to be able to upload server descriptors so clients know we're still
+   * running, and download directories so we can detect if we're obsolete.
+   * Leave control conns because we still want to be controllable.
+   */
+  while ((conn = connection_get_by_type(CONN_TYPE_OR)) ||
+         (conn = connection_get_by_type(CONN_TYPE_AP)) ||
+         (conn = connection_get_by_type(CONN_TYPE_EXIT))) {
+    if (CONN_IS_EDGE(conn)) {
+      connection_edge_end(TO_EDGE_CONN(conn), END_STREAM_REASON_HIBERNATING);
+    }
+    log_info(LD_NET,"Closing conn type %d", conn->type);
+    if (conn->type == CONN_TYPE_AP) {
+      /* send socks failure if needed */
+      connection_mark_unattached_ap(TO_ENTRY_CONN(conn),
+                                    END_STREAM_REASON_HIBERNATING);
+    } else if (conn->type == CONN_TYPE_OR) {
+      if (TO_OR_CONN(conn)->chan) {
+        connection_or_close_normally(TO_OR_CONN(conn), 0);
+      } else {
+         connection_mark_for_close(conn);
+      }
+    } else {
+      connection_mark_for_close(conn);
+    }
+  }
+
+  if (now < interval_wakeup_time)
+    hibernate_end_time = interval_wakeup_time;
+  else
+    hibernate_end_time = interval_end_time;
+
+  accounting_record_bandwidth_usage(now, get_or_state());
+
+  or_state_mark_dirty(get_or_state(),
+                      get_options()->AvoidDiskWrites ? now+600 : 0);
+
+  hibernate_schedule_wakeup_event(now, hibernate_end_time);
+}
+
+/**
+ * Schedule a mainloop event at <b>end_time</b> to wake up from a dormant
+ * state.  We can't rely on this happening from second_elapsed_callback,
+ * since second_elapsed_callback will be shut down when we're dormant.
+ *
+ * (Note that We might immediately go back to sleep after we set the next
+ * wakeup time.)
+ */
+static void
+hibernate_schedule_wakeup_event(time_t now, time_t end_time)
+{
+  struct timeval delay = { 0, 0 };
+
+  if (now >= end_time) {
+    // In these cases we always wait at least a second, to avoid running
+    // the callback in a tight loop.
+    delay.tv_sec = 1;
+  } else {
+    delay.tv_sec = (end_time - now);
+  }
+
+  if (!wakeup_event) {
+    wakeup_event = mainloop_event_postloop_new(wakeup_event_callback, NULL);
+  }
+
+  mainloop_event_schedule(wakeup_event, &delay);
+}
+
+/**
+ * Called at the end of the interval, or at the wakeup time of the current
+ * interval, to exit the dormant state.
+ **/
+static void
+wakeup_event_callback(mainloop_event_t *ev, void *data)
+{
+  (void) ev;
+  (void) data;
+
+  const time_t now = time(NULL);
+  accounting_run_housekeeping(now);
+  consider_hibernation(now);
+  if (hibernate_state != HIBERNATE_STATE_DORMANT) {
+    /* We woke up, so everything's great here */
+    return;
+  }
+
+  /* We're still dormant. */
+  if (now < interval_wakeup_time)
+    hibernate_end_time = interval_wakeup_time;
+  else
+    hibernate_end_time = interval_end_time;
+
+  hibernate_schedule_wakeup_event(now, hibernate_end_time);
+}
+
+/** Called when hibernate_end_time has arrived. */
+static void
+hibernate_end_time_elapsed(time_t now)
+{
+  char buf[ISO_TIME_LEN+1];
+
+  /* The interval has ended, or it is wakeup time.  Find out which. */
+  accounting_run_housekeeping(now);
+  if (interval_wakeup_time <= now) {
+    /* The interval hasn't changed, but interval_wakeup_time has passed.
+     * It's time to wake up and start being a server. */
+    hibernate_end(HIBERNATE_STATE_LIVE);
+    return;
+  } else {
+    /* The interval has changed, and it isn't time to wake up yet. */
+    hibernate_end_time = interval_wakeup_time;
+    format_iso_time(buf,interval_wakeup_time);
+    if (hibernate_state != HIBERNATE_STATE_DORMANT) {
+      /* We weren't sleeping before; we should sleep now. */
+      log_notice(LD_ACCT,
+                 "Accounting period ended. Commencing hibernation until "
+                 "%s UTC", buf);
+      hibernate_go_dormant(now);
+    } else {
+      log_notice(LD_ACCT,
+             "Accounting period ended. This period, we will hibernate"
+             " until %s UTC",buf);
+    }
+  }
+}
+
+/** Consider our environment and decide if it's time
+ * to start/stop hibernating.
+ */
+void
+consider_hibernation(time_t now)
+{
+  int accounting_enabled = get_options()->AccountingMax != 0;
+  char buf[ISO_TIME_LEN+1];
+  hibernate_state_t prev_state = hibernate_state;
+
+  /* If we're in 'exiting' mode, then we just shut down after the interval
+   * elapses. */
+  if (hibernate_state == HIBERNATE_STATE_EXITING) {
+    tor_assert(shutdown_time);
+    if (shutdown_time <= now) {
+      log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
+      tor_shutdown_event_loop_and_exit(0);
+    }
+    return; /* if exiting soon, don't worry about bandwidth limits */
+  }
+
+  if (hibernate_state == HIBERNATE_STATE_DORMANT) {
+    /* We've been hibernating because of bandwidth accounting. */
+    tor_assert(hibernate_end_time);
+    if (hibernate_end_time > now && accounting_enabled) {
+      /* If we're hibernating, don't wake up until it's time, regardless of
+       * whether we're in a new interval. */
+      return ;
+    } else {
+      hibernate_end_time_elapsed(now);
+    }
+  }
+
+  /* Else, we aren't hibernating. See if it's time to start hibernating, or to
+   * go dormant. */
+  if (hibernate_state == HIBERNATE_STATE_LIVE ||
+      hibernate_state == HIBERNATE_STATE_INITIAL) {
+    if (hibernate_soft_limit_reached()) {
+      log_notice(LD_ACCT,
+                 "Bandwidth soft limit reached; commencing hibernation. "
+                 "No new connections will be accepted");
+      hibernate_begin(HIBERNATE_STATE_LOWBANDWIDTH, now);
+    } else if (accounting_enabled && now < interval_wakeup_time) {
+      format_local_iso_time(buf,interval_wakeup_time);
+      log_notice(LD_ACCT,
+                 "Commencing hibernation. We will wake up at %s local time.",
+                 buf);
+      hibernate_go_dormant(now);
+    } else if (hibernate_state == HIBERNATE_STATE_INITIAL) {
+      hibernate_end(HIBERNATE_STATE_LIVE);
+    }
+  }
+
+  if (hibernate_state == HIBERNATE_STATE_LOWBANDWIDTH) {
+    if (!accounting_enabled) {
+      hibernate_end_time_elapsed(now);
+    } else if (hibernate_hard_limit_reached()) {
+      hibernate_go_dormant(now);
+    } else if (hibernate_end_time <= now) {
+      /* The hibernation period ended while we were still in lowbandwidth.*/
+      hibernate_end_time_elapsed(now);
+    }
+  }
+
+  /* Dispatch a controller event if the hibernation state changed. */
+  if (hibernate_state != prev_state)
+    on_hibernate_state_change(prev_state);
+}
+
+/** Helper function: called when we get a GETINFO request for an
+ * accounting-related key on the control connection <b>conn</b>.  If we can
+ * answer the request for <b>question</b>, then set *<b>answer</b> to a newly
+ * allocated string holding the result.  Otherwise, set *<b>answer</b> to
+ * NULL. */
+int
+getinfo_helper_accounting(control_connection_t *conn,
+                          const char *question, char **answer,
+                          const char **errmsg)
+{
+  (void) conn;
+  (void) errmsg;
+  if (!strcmp(question, "accounting/enabled")) {
+    *answer = tor_strdup(accounting_is_enabled(get_options()) ? "1" : "0");
+  } else if (!strcmp(question, "accounting/hibernating")) {
+    *answer = tor_strdup(hibernate_state_to_string(hibernate_state));
+    tor_strlower(*answer);
+  } else if (!strcmp(question, "accounting/bytes")) {
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                 (n_bytes_read_in_interval),
+                 (n_bytes_written_in_interval));
+  } else if (!strcmp(question, "accounting/bytes-left")) {
+    uint64_t limit = get_options()->AccountingMax;
+    if (get_options()->AccountingRule == ACCT_SUM) {
+      uint64_t total_left = 0;
+      uint64_t total_bytes = get_accounting_bytes();
+      if (total_bytes < limit)
+        total_left = limit - total_bytes;
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                   (total_left), (total_left));
+    } else if (get_options()->AccountingRule == ACCT_IN) {
+      uint64_t read_left = 0;
+      if (n_bytes_read_in_interval < limit)
+        read_left = limit - n_bytes_read_in_interval;
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                   (read_left), (limit));
+    } else if (get_options()->AccountingRule == ACCT_OUT) {
+      uint64_t write_left = 0;
+      if (n_bytes_written_in_interval < limit)
+        write_left = limit - n_bytes_written_in_interval;
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                   (limit), (write_left));
+    } else {
+      uint64_t read_left = 0, write_left = 0;
+      if (n_bytes_read_in_interval < limit)
+        read_left = limit - n_bytes_read_in_interval;
+      if (n_bytes_written_in_interval < limit)
+        write_left = limit - n_bytes_written_in_interval;
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                   (read_left), (write_left));
+    }
+  } else if (!strcmp(question, "accounting/interval-start")) {
+    *answer = tor_malloc(ISO_TIME_LEN+1);
+    format_iso_time(*answer, interval_start_time);
+  } else if (!strcmp(question, "accounting/interval-wake")) {
+    *answer = tor_malloc(ISO_TIME_LEN+1);
+    format_iso_time(*answer, interval_wakeup_time);
+  } else if (!strcmp(question, "accounting/interval-end")) {
+    *answer = tor_malloc(ISO_TIME_LEN+1);
+    format_iso_time(*answer, interval_end_time);
+  } else {
+    *answer = NULL;
+  }
+  return 0;
+}
+
+/**
+ * Helper function: called when the hibernation state changes, and sends a
+ * SERVER_STATUS event to notify interested controllers of the accounting
+ * state change.
+ */
+static void
+on_hibernate_state_change(hibernate_state_t prev_state)
+{
+  control_event_server_status(LOG_NOTICE,
+                              "HIBERNATION_STATUS STATUS=%s",
+                              hibernate_state_to_string(hibernate_state));
+
+  /* We are changing hibernation state, this can affect the main loop event
+   * list. Rescan it to update the events state. We do this whatever the new
+   * hibernation state because they can each possibly affect an event. The
+   * initial state means we are booting up so we shouldn't scan here because
+   * at this point the events in the list haven't been initialized. */
+  if (prev_state != HIBERNATE_STATE_INITIAL) {
+    rescan_periodic_events(get_options());
+  }
+
+  reschedule_per_second_timer();
+}
+
+/** Free all resources held by the accounting module */
+void
+accounting_free_all(void)
+{
+  mainloop_event_free(wakeup_event);
+  hibernate_state = HIBERNATE_STATE_INITIAL;
+  hibernate_end_time = 0;
+  shutdown_time = 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Manually change the hibernation state.  Private; used only by the unit
+ * tests.
+ */
+void
+hibernate_set_state_for_testing_(hibernate_state_t newstate)
+{
+  hibernate_state = newstate;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/hibernate/hibernate.h b/src/feature/hibernate/hibernate.h
new file mode 100644
index 0000000000..bfd8571cd6
--- /dev/null
+++ b/src/feature/hibernate/hibernate.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hibernate.h
+ * \brief Header file for hibernate.c.
+ **/
+
+#ifndef TOR_HIBERNATE_H
+#define TOR_HIBERNATE_H
+
+#include "lib/testsupport/testsupport.h"
+
+int accounting_parse_options(const or_options_t *options, int validate_only);
+MOCK_DECL(int, accounting_is_enabled, (const or_options_t *options));
+int accounting_get_interval_length(void);
+MOCK_DECL(time_t, accounting_get_end_time, (void));
+void configure_accounting(time_t now);
+uint64_t get_accounting_bytes(void);
+void accounting_run_housekeeping(time_t now);
+void accounting_add_bytes(size_t n_read, size_t n_written, int seconds);
+int accounting_record_bandwidth_usage(time_t now, or_state_t *state);
+void hibernate_begin_shutdown(void);
+MOCK_DECL(int, we_are_hibernating, (void));
+MOCK_DECL(int, we_are_fully_hibernating,(void));
+void consider_hibernation(time_t now);
+int getinfo_helper_accounting(control_connection_t *conn,
+                              const char *question, char **answer,
+                              const char **errmsg);
+uint64_t get_accounting_max_total(void);
+void accounting_free_all(void);
+
+#ifdef HIBERNATE_PRIVATE
+/** Possible values of hibernate_state */
+typedef enum {
+  /** We are running normally. */
+  HIBERNATE_STATE_LIVE=1,
+  /** We're trying to shut down cleanly, and we'll kill all active connections
+   * at shutdown_time. */
+  HIBERNATE_STATE_EXITING=2,
+  /** We're running low on allocated bandwidth for this period, so we won't
+   * accept any new connections. */
+  HIBERNATE_STATE_LOWBANDWIDTH=3,
+  /** We are hibernating, and we won't wake up till there's more bandwidth to
+   * use. */
+  HIBERNATE_STATE_DORMANT=4,
+  /** We start out in state default, which means we havent decided which state
+   * we're in. */
+  HIBERNATE_STATE_INITIAL=5
+} hibernate_state_t;
+
+#ifdef TOR_UNIT_TESTS
+void hibernate_set_state_for_testing_(hibernate_state_t newstate);
+#endif
+#endif /* defined(HIBERNATE_PRIVATE) */
+
+#endif /* !defined(TOR_HIBERNATE_H) */
+
diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c
new file mode 100644
index 0000000000..3772e0c0ed
--- /dev/null
+++ b/src/feature/hs/hs_cache.c
@@ -0,0 +1,979 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cache.c
+ * \brief Handle hidden service descriptor caches.
+ **/
+
+/* For unit tests.*/
+#define HS_CACHE_PRIVATE
+
+#include "or/or.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/hs_ident.h"
+#include "or/hs_common.h"
+#include "or/hs_client.h"
+#include "or/hs_descriptor.h"
+#include "or/networkstatus.h"
+#include "or/rendcache.h"
+
+#include "or/hs_cache.h"
+
+#include "or/networkstatus_st.h"
+
+static int cached_client_descriptor_has_expired(time_t now,
+           const hs_cache_client_descriptor_t *cached_desc);
+
+/********************** Directory HS cache ******************/
+
+/* Directory descriptor cache. Map indexed by blinded key. */
+static digest256map_t *hs_cache_v3_dir;
+
+/* Remove a given descriptor from our cache. */
+static void
+remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_remove(hs_cache_v3_dir, desc->key);
+}
+
+/* Store a given descriptor in our cache. */
+static void
+store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_set(hs_cache_v3_dir, desc->key, desc);
+}
+
+/* Query our cache and return the entry or NULL if not found. */
+static hs_cache_dir_descriptor_t *
+lookup_v3_desc_as_dir(const uint8_t *key)
+{
+  tor_assert(key);
+  return digest256map_get(hs_cache_v3_dir, key);
+}
+
+#define cache_dir_desc_free(val) \
+  FREE_AND_NULL(hs_cache_dir_descriptor_t, cache_dir_desc_free_, (val))
+
+/* Free a directory descriptor object. */
+static void
+cache_dir_desc_free_(hs_cache_dir_descriptor_t *desc)
+{
+  if (desc == NULL) {
+    return;
+  }
+  hs_desc_plaintext_data_free(desc->plaintext_data);
+  tor_free(desc->encoded_desc);
+  tor_free(desc);
+}
+
+/* Helper function: Use by the free all function using the digest256map
+ * interface to cache entries. */
+static void
+cache_dir_desc_free_void(void *ptr)
+{
+  cache_dir_desc_free_(ptr);
+}
+
+/* Create a new directory cache descriptor object from a encoded descriptor.
+ * On success, return the heap-allocated cache object, otherwise return NULL if
+ * we can't decode the descriptor. */
+static hs_cache_dir_descriptor_t *
+cache_dir_desc_new(const char *desc)
+{
+  hs_cache_dir_descriptor_t *dir_desc;
+
+  tor_assert(desc);
+
+  dir_desc = tor_malloc_zero(sizeof(hs_cache_dir_descriptor_t));
+  dir_desc->plaintext_data =
+    tor_malloc_zero(sizeof(hs_desc_plaintext_data_t));
+  dir_desc->encoded_desc = tor_strdup(desc);
+
+  if (hs_desc_decode_plaintext(desc, dir_desc->plaintext_data) < 0) {
+    log_debug(LD_DIR, "Unable to decode descriptor. Rejecting.");
+    goto err;
+  }
+
+  /* The blinded pubkey is the indexed key. */
+  dir_desc->key = dir_desc->plaintext_data->blinded_pubkey.pubkey;
+  dir_desc->created_ts = time(NULL);
+  return dir_desc;
+
+ err:
+  cache_dir_desc_free(dir_desc);
+  return NULL;
+}
+
+/* Return the size of a cache entry in bytes. */
+static size_t
+cache_get_dir_entry_size(const hs_cache_dir_descriptor_t *entry)
+{
+  return (sizeof(*entry) + hs_desc_plaintext_obj_size(entry->plaintext_data)
+          + strlen(entry->encoded_desc));
+}
+
+/* Try to store a valid version 3 descriptor in the directory cache. Return 0
+ * on success else a negative value is returned indicating that we have a
+ * newer version in our cache. On error, caller is responsible to free the
+ * given descriptor desc. */
+static int
+cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc)
+{
+  hs_cache_dir_descriptor_t *cache_entry;
+
+  tor_assert(desc);
+
+  /* Verify if we have an entry in the cache for that key and if yes, check
+   * if we should replace it? */
+  cache_entry = lookup_v3_desc_as_dir(desc->key);
+  if (cache_entry != NULL) {
+    /* Only replace descriptor if revision-counter is greater than the one
+     * in our cache */
+    if (cache_entry->plaintext_data->revision_counter >=
+        desc->plaintext_data->revision_counter) {
+      log_info(LD_REND, "Descriptor revision counter in our cache is "
+               "greater or equal than the one we received (%d/%d). "
+               "Rejecting!",
+               (int)cache_entry->plaintext_data->revision_counter,
+               (int)desc->plaintext_data->revision_counter);
+      goto err;
+    }
+    /* We now know that the descriptor we just received is a new one so
+     * remove the entry we currently have from our cache so we can then
+     * store the new one. */
+    remove_v3_desc_as_dir(cache_entry);
+    rend_cache_decrement_allocation(cache_get_dir_entry_size(cache_entry));
+    cache_dir_desc_free(cache_entry);
+  }
+  /* Store the descriptor we just got. We are sure here that either we
+   * don't have the entry or we have a newer descriptor and the old one
+   * has been removed from the cache. */
+  store_v3_desc_as_dir(desc);
+
+  /* Update our total cache size with this entry for the OOM. This uses the
+   * old HS protocol cache subsystem for which we are tied with. */
+  rend_cache_increment_allocation(cache_get_dir_entry_size(desc));
+
+  /* XXX: Update HS statistics. We should have specific stats for v3. */
+
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Using the query which is the base64 encoded blinded key of a version 3
+ * descriptor, lookup in our directory cache the entry. If found, 1 is
+ * returned and desc_out is populated with a newly allocated string being the
+ * encoded descriptor. If not found, 0 is returned and desc_out is untouched.
+ * On error, a negative value is returned and desc_out is untouched. */
+static int
+cache_lookup_v3_as_dir(const char *query, const char **desc_out)
+{
+  int found = 0;
+  ed25519_public_key_t blinded_key;
+  const hs_cache_dir_descriptor_t *entry;
+
+  tor_assert(query);
+
+  /* Decode blinded key using the given query value. */
+  if (ed25519_public_from_base64(&blinded_key, query) < 0) {
+    log_info(LD_REND, "Unable to decode the v3 HSDir query %s.",
+             safe_str_client(query));
+    goto err;
+  }
+
+  entry = lookup_v3_desc_as_dir(blinded_key.pubkey);
+  if (entry != NULL) {
+    found = 1;
+    if (desc_out) {
+      *desc_out = entry->encoded_desc;
+    }
+  }
+
+  return found;
+
+ err:
+  return -1;
+}
+
+/* Clean the v3 cache by removing any entry that has expired using the
+ * <b>global_cutoff</b> value. If <b>global_cutoff</b> is 0, the cleaning
+ * process will use the lifetime found in the plaintext data section. Return
+ * the number of bytes cleaned. */
+STATIC size_t
+cache_clean_v3_as_dir(time_t now, time_t global_cutoff)
+{
+  size_t bytes_removed = 0;
+
+  /* Code flow error if this ever happens. */
+  tor_assert(global_cutoff >= 0);
+
+  if (!hs_cache_v3_dir) { /* No cache to clean. Just return. */
+    return 0;
+  }
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_dir, key,
+                              hs_cache_dir_descriptor_t *, entry) {
+    size_t entry_size;
+    time_t cutoff = global_cutoff;
+    if (!cutoff) {
+      /* Cutoff is the lifetime of the entry found in the descriptor. */
+      cutoff = now - entry->plaintext_data->lifetime_sec;
+    }
+
+    /* If the entry has been created _after_ the cutoff, not expired so
+     * continue to the next entry in our v3 cache. */
+    if (entry->created_ts > cutoff) {
+      continue;
+    }
+    /* Here, our entry has expired, remove and free. */
+    MAP_DEL_CURRENT(key);
+    entry_size = cache_get_dir_entry_size(entry);
+    bytes_removed += entry_size;
+    /* Entry is not in the cache anymore, destroy it. */
+    cache_dir_desc_free(entry);
+    /* Update our cache entry allocation size for the OOM. */
+    rend_cache_decrement_allocation(entry_size);
+    /* Logging. */
+    {
+      char key_b64[BASE64_DIGEST256_LEN + 1];
+      digest256_to_base64(key_b64, (const char *) key);
+      log_info(LD_REND, "Removing v3 descriptor '%s' from HSDir cache",
+               safe_str_client(key_b64));
+    }
+  } DIGEST256MAP_FOREACH_END;
+
+  return bytes_removed;
+}
+
+/* Given an encoded descriptor, store it in the directory cache depending on
+ * which version it is. Return a negative value on error. On success, 0 is
+ * returned. */
+int
+hs_cache_store_as_dir(const char *desc)
+{
+  hs_cache_dir_descriptor_t *dir_desc = NULL;
+
+  tor_assert(desc);
+
+  /* Create a new cache object. This can fail if the descriptor plaintext data
+   * is unparseable which in this case a log message will be triggered. */
+  dir_desc = cache_dir_desc_new(desc);
+  if (dir_desc == NULL) {
+    goto err;
+  }
+
+  /* Call the right function against the descriptor version. At this point,
+   * we are sure that the descriptor's version is supported else the
+   * decoding would have failed. */
+  switch (dir_desc->plaintext_data->version) {
+  case HS_VERSION_THREE:
+  default:
+    if (cache_store_v3_as_dir(dir_desc) < 0) {
+      goto err;
+    }
+    break;
+  }
+  return 0;
+
+ err:
+  cache_dir_desc_free(dir_desc);
+  return -1;
+}
+
+/* Using the query, lookup in our directory cache the entry. If found, 1 is
+ * returned and desc_out is populated with a newly allocated string being
+ * the encoded descriptor. If not found, 0 is returned and desc_out is
+ * untouched. On error, a negative value is returned and desc_out is
+ * untouched. */
+int
+hs_cache_lookup_as_dir(uint32_t version, const char *query,
+                       const char **desc_out)
+{
+  int found;
+
+  tor_assert(query);
+  /* This should never be called with an unsupported version. */
+  tor_assert(hs_desc_is_supported_version(version));
+
+  switch (version) {
+  case HS_VERSION_THREE:
+  default:
+    found = cache_lookup_v3_as_dir(query, desc_out);
+    break;
+  }
+
+  return found;
+}
+
+/* Clean all directory caches using the current time now. */
+void
+hs_cache_clean_as_dir(time_t now)
+{
+  time_t cutoff;
+
+  /* Start with v2 cache cleaning. */
+  cutoff = now - rend_cache_max_entry_lifetime();
+  rend_cache_clean_v2_descs_as_dir(cutoff);
+
+  /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function
+   * to compute the cutoff by itself using the lifetime value. */
+  cache_clean_v3_as_dir(now, 0);
+}
+
+/********************** Client-side HS cache ******************/
+
+/* Client-side HS descriptor cache. Map indexed by service identity key. */
+static digest256map_t *hs_cache_v3_client;
+
+/* Client-side introduction point state cache. Map indexed by service public
+ * identity key (onion address). It contains hs_cache_client_intro_state_t
+ * objects all related to a specific service. */
+static digest256map_t *hs_cache_client_intro_state;
+
+/* Return the size of a client cache entry in bytes. */
+static size_t
+cache_get_client_entry_size(const hs_cache_client_descriptor_t *entry)
+{
+  return sizeof(*entry) +
+         strlen(entry->encoded_desc) + hs_desc_obj_size(entry->desc);
+}
+
+/* Remove a given descriptor from our cache. */
+static void
+remove_v3_desc_as_client(const hs_cache_client_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_remove(hs_cache_v3_client, desc->key.pubkey);
+  /* Update cache size with this entry for the OOM handler. */
+  rend_cache_decrement_allocation(cache_get_client_entry_size(desc));
+}
+
+/* Store a given descriptor in our cache. */
+static void
+store_v3_desc_as_client(hs_cache_client_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_set(hs_cache_v3_client, desc->key.pubkey, desc);
+  /* Update cache size with this entry for the OOM handler. */
+  rend_cache_increment_allocation(cache_get_client_entry_size(desc));
+}
+
+/* Query our cache and return the entry or NULL if not found or if expired. */
+STATIC hs_cache_client_descriptor_t *
+lookup_v3_desc_as_client(const uint8_t *key)
+{
+  time_t now = approx_time();
+  hs_cache_client_descriptor_t *cached_desc;
+
+  tor_assert(key);
+
+  /* Do the lookup */
+  cached_desc = digest256map_get(hs_cache_v3_client, key);
+  if (!cached_desc) {
+    return NULL;
+  }
+
+  /* Don't return expired entries */
+  if (cached_client_descriptor_has_expired(now, cached_desc)) {
+    return NULL;
+  }
+
+  return cached_desc;
+}
+
+/* Parse the encoded descriptor in <b>desc_str</b> using
+ * <b>service_identity_pk<b> to decrypt it first.
+ *
+ * If everything goes well, allocate and return a new
+ * hs_cache_client_descriptor_t object. In case of error, return NULL. */
+static hs_cache_client_descriptor_t *
+cache_client_desc_new(const char *desc_str,
+                      const ed25519_public_key_t *service_identity_pk)
+{
+  hs_descriptor_t *desc = NULL;
+  hs_cache_client_descriptor_t *client_desc = NULL;
+
+  tor_assert(desc_str);
+  tor_assert(service_identity_pk);
+
+  /* Decode the descriptor we just fetched. */
+  if (hs_client_decode_descriptor(desc_str, service_identity_pk, &desc) < 0) {
+    goto end;
+  }
+  tor_assert(desc);
+
+  /* All is good: make a cache object for this descriptor */
+  client_desc = tor_malloc_zero(sizeof(hs_cache_client_descriptor_t));
+  ed25519_pubkey_copy(&client_desc->key, service_identity_pk);
+  /* Set expiration time for this cached descriptor to be the start of the next
+   * time period since that's when clients need to start using the next blinded
+   * pk of the service (and hence will need its next descriptor). */
+  client_desc->expiration_ts = hs_get_start_time_of_next_time_period(0);
+  client_desc->desc = desc;
+  client_desc->encoded_desc = tor_strdup(desc_str);
+
+ end:
+  return client_desc;
+}
+
+#define cache_client_desc_free(val) \
+  FREE_AND_NULL(hs_cache_client_descriptor_t, cache_client_desc_free_, (val))
+
+/** Free memory allocated by <b>desc</b>. */
+static void
+cache_client_desc_free_(hs_cache_client_descriptor_t *desc)
+{
+  if (desc == NULL) {
+    return;
+  }
+  hs_descriptor_free(desc->desc);
+  memwipe(&desc->key, 0, sizeof(desc->key));
+  memwipe(desc->encoded_desc, 0, strlen(desc->encoded_desc));
+  tor_free(desc->encoded_desc);
+  tor_free(desc);
+}
+
+/** Helper function: Use by the free all function to clear the client cache */
+static void
+cache_client_desc_free_void(void *ptr)
+{
+  hs_cache_client_descriptor_t *desc = ptr;
+  cache_client_desc_free(desc);
+}
+
+/* Return a newly allocated and initialized hs_cache_intro_state_t object. */
+static hs_cache_intro_state_t *
+cache_intro_state_new(void)
+{
+  hs_cache_intro_state_t *state = tor_malloc_zero(sizeof(*state));
+  state->created_ts = approx_time();
+  return state;
+}
+
+#define cache_intro_state_free(val) \
+  FREE_AND_NULL(hs_cache_intro_state_t, cache_intro_state_free_, (val))
+
+/* Free an hs_cache_intro_state_t object. */
+static void
+cache_intro_state_free_(hs_cache_intro_state_t *state)
+{
+  tor_free(state);
+}
+
+/* Helper function: use by the free all function. */
+static void
+cache_intro_state_free_void(void *state)
+{
+  cache_intro_state_free_(state);
+}
+
+/* Return a newly allocated and initialized hs_cache_client_intro_state_t
+ * object. */
+static hs_cache_client_intro_state_t *
+cache_client_intro_state_new(void)
+{
+  hs_cache_client_intro_state_t *cache = tor_malloc_zero(sizeof(*cache));
+  cache->intro_points = digest256map_new();
+  return cache;
+}
+
+#define cache_client_intro_state_free(val)              \
+  FREE_AND_NULL(hs_cache_client_intro_state_t,          \
+                cache_client_intro_state_free_, (val))
+
+/* Free a cache client intro state object. */
+static void
+cache_client_intro_state_free_(hs_cache_client_intro_state_t *cache)
+{
+  if (cache == NULL) {
+    return;
+  }
+  digest256map_free(cache->intro_points, cache_intro_state_free_void);
+  tor_free(cache);
+}
+
+/* Helper function: use by the free all function. */
+static void
+cache_client_intro_state_free_void(void *entry)
+{
+  cache_client_intro_state_free_(entry);
+}
+
+/* For the given service identity key service_pk and an introduction
+ * authentication key auth_key, lookup the intro state object. Return 1 if
+ * found and put it in entry if not NULL. Return 0 if not found and entry is
+ * untouched. */
+static int
+cache_client_intro_state_lookup(const ed25519_public_key_t *service_pk,
+                                const ed25519_public_key_t *auth_key,
+                                hs_cache_intro_state_t **entry)
+{
+  hs_cache_intro_state_t *state;
+  hs_cache_client_intro_state_t *cache;
+
+  tor_assert(service_pk);
+  tor_assert(auth_key);
+
+  /* Lookup the intro state cache for this service key. */
+  cache = digest256map_get(hs_cache_client_intro_state, service_pk->pubkey);
+  if (cache == NULL) {
+    goto not_found;
+  }
+
+  /* From the cache we just found for the service, lookup in the introduction
+   * points map for the given authentication key. */
+  state = digest256map_get(cache->intro_points, auth_key->pubkey);
+  if (state == NULL) {
+    goto not_found;
+  }
+  if (entry) {
+    *entry = state;
+  }
+  return 1;
+ not_found:
+  return 0;
+}
+
+/* Note the given failure in state. */
+static void
+cache_client_intro_state_note(hs_cache_intro_state_t *state,
+                              rend_intro_point_failure_t failure)
+{
+  tor_assert(state);
+  switch (failure) {
+  case INTRO_POINT_FAILURE_GENERIC:
+    state->error = 1;
+    break;
+  case INTRO_POINT_FAILURE_TIMEOUT:
+    state->timed_out = 1;
+    break;
+  case INTRO_POINT_FAILURE_UNREACHABLE:
+    state->unreachable_count++;
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+    return;
+  }
+}
+
+/* For the given service identity key service_pk and an introduction
+ * authentication key auth_key, add an entry in the client intro state cache
+ * If no entry exists for the service, it will create one. If state is non
+ * NULL, it will point to the new intro state entry. */
+static void
+cache_client_intro_state_add(const ed25519_public_key_t *service_pk,
+                             const ed25519_public_key_t *auth_key,
+                             hs_cache_intro_state_t **state)
+{
+  hs_cache_intro_state_t *entry, *old_entry;
+  hs_cache_client_intro_state_t *cache;
+
+  tor_assert(service_pk);
+  tor_assert(auth_key);
+
+  /* Lookup the state cache for this service key. */
+  cache = digest256map_get(hs_cache_client_intro_state, service_pk->pubkey);
+  if (cache == NULL) {
+    cache = cache_client_intro_state_new();
+    digest256map_set(hs_cache_client_intro_state, service_pk->pubkey, cache);
+  }
+
+  entry = cache_intro_state_new();
+  old_entry = digest256map_set(cache->intro_points, auth_key->pubkey, entry);
+  /* This should never happened because the code flow is to lookup the entry
+   * before adding it. But, just in case, non fatal assert and free it. */
+  tor_assert_nonfatal(old_entry == NULL);
+  tor_free(old_entry);
+
+  if (state) {
+    *state = entry;
+  }
+}
+
+/* Remove every intro point state entry from cache that has been created
+ * before or at the cutoff. */
+static void
+cache_client_intro_state_clean(time_t cutoff,
+                               hs_cache_client_intro_state_t *cache)
+{
+  tor_assert(cache);
+
+  DIGEST256MAP_FOREACH_MODIFY(cache->intro_points, key,
+                              hs_cache_intro_state_t *, entry) {
+    if (entry->created_ts <= cutoff) {
+      cache_intro_state_free(entry);
+      MAP_DEL_CURRENT(key);
+    }
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Return true iff no intro points are in this cache. */
+static int
+cache_client_intro_state_is_empty(const hs_cache_client_intro_state_t *cache)
+{
+  return digest256map_isempty(cache->intro_points);
+}
+
+/** Check whether <b>client_desc</b> is useful for us, and store it in the
+ *  client-side HS cache if so. The client_desc is freed if we already have a
+ *  fresher (higher revision counter count) in the cache. */
+static int
+cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
+{
+  hs_cache_client_descriptor_t *cache_entry;
+
+  /* TODO: Heavy code duplication with cache_store_as_dir(). Consider
+   * refactoring and uniting! */
+
+  tor_assert(client_desc);
+
+  /* Check if we already have a descriptor from this HS in cache. If we do,
+   * check if this descriptor is newer than the cached one */
+  cache_entry = lookup_v3_desc_as_client(client_desc->key.pubkey);
+  if (cache_entry != NULL) {
+    /* If we have an entry in our cache that has a revision counter greater
+     * than the one we just fetched, discard the one we fetched. */
+    if (cache_entry->desc->plaintext_data.revision_counter >
+        client_desc->desc->plaintext_data.revision_counter) {
+      cache_client_desc_free(client_desc);
+      goto done;
+    }
+    /* Remove old entry. Make space for the new one! */
+    remove_v3_desc_as_client(cache_entry);
+    cache_client_desc_free(cache_entry);
+  }
+
+  /* Store descriptor in cache */
+  store_v3_desc_as_client(client_desc);
+
+ done:
+  return 0;
+}
+
+/* Return true iff the cached client descriptor at <b>cached_desc</b has
+ * expired. */
+static int
+cached_client_descriptor_has_expired(time_t now,
+                               const hs_cache_client_descriptor_t *cached_desc)
+{
+  /* We use the current consensus time to see if we should expire this
+   * descriptor since we use consensus time for all other parts of the protocol
+   * as well (e.g. to build the blinded key and compute time periods). */
+  const networkstatus_t *ns = networkstatus_get_live_consensus(now);
+  /* If we don't have a recent consensus, consider this entry expired since we
+   * will want to fetch a new HS desc when we get a live consensus. */
+  if (!ns) {
+    return 1;
+  }
+
+  if (cached_desc->expiration_ts <= ns->valid_after) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/* clean the client cache using now as the current time. Return the total size
+ * of removed bytes from the cache. */
+static size_t
+cache_clean_v3_as_client(time_t now)
+{
+  size_t bytes_removed = 0;
+
+  if (!hs_cache_v3_client) { /* No cache to clean. Just return. */
+    return 0;
+  }
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_client, key,
+                              hs_cache_client_descriptor_t *, entry) {
+    size_t entry_size;
+
+    /* If the entry has not expired, continue to the next cached entry */
+    if (!cached_client_descriptor_has_expired(now, entry)) {
+      continue;
+    }
+    /* Here, our entry has expired, remove and free. */
+    MAP_DEL_CURRENT(key);
+    entry_size = cache_get_client_entry_size(entry);
+    bytes_removed += entry_size;
+    /* Entry is not in the cache anymore, destroy it. */
+    cache_client_desc_free(entry);
+    /* Update our OOM. We didn't use the remove() function because we are in
+     * a loop so we have to explicitly decrement. */
+    rend_cache_decrement_allocation(entry_size);
+    /* Logging. */
+    {
+      char key_b64[BASE64_DIGEST256_LEN + 1];
+      digest256_to_base64(key_b64, (const char *) key);
+      log_info(LD_REND, "Removing hidden service v3 descriptor '%s' "
+                        "from client cache",
+               safe_str_client(key_b64));
+    }
+  } DIGEST256MAP_FOREACH_END;
+
+  return bytes_removed;
+}
+
+/** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
+ *  its HS encoded descriptor if it's stored in our cache, or NULL if not. */
+const char *
+hs_cache_lookup_encoded_as_client(const ed25519_public_key_t *key)
+{
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(key);
+
+  cached_desc = lookup_v3_desc_as_client(key->pubkey);
+  if (cached_desc) {
+    tor_assert(cached_desc->encoded_desc);
+    return cached_desc->encoded_desc;
+  }
+
+  return NULL;
+}
+
+/** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
+ *  its HS descriptor if it's stored in our cache, or NULL if not. */
+const hs_descriptor_t *
+hs_cache_lookup_as_client(const ed25519_public_key_t *key)
+{
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(key);
+
+  cached_desc = lookup_v3_desc_as_client(key->pubkey);
+  if (cached_desc) {
+    tor_assert(cached_desc->desc);
+    return cached_desc->desc;
+  }
+
+  return NULL;
+}
+
+/** Public API: Given an encoded descriptor, store it in the client HS
+ *  cache. Return -1 on error, 0 on success .*/
+int
+hs_cache_store_as_client(const char *desc_str,
+                         const ed25519_public_key_t *identity_pk)
+{
+  hs_cache_client_descriptor_t *client_desc = NULL;
+
+  tor_assert(desc_str);
+  tor_assert(identity_pk);
+
+  /* Create client cache descriptor object */
+  client_desc = cache_client_desc_new(desc_str, identity_pk);
+  if (!client_desc) {
+    log_warn(LD_GENERAL, "Failed to parse received descriptor %s.",
+             escaped(desc_str));
+    goto err;
+  }
+
+  /* Push it to the cache */
+  if (cache_store_as_client(client_desc) < 0) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  cache_client_desc_free(client_desc);
+  return -1;
+}
+
+/* Clean all client caches using the current time now. */
+void
+hs_cache_clean_as_client(time_t now)
+{
+  /* Start with v2 cache cleaning. */
+  rend_cache_clean(now, REND_CACHE_TYPE_CLIENT);
+  /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function
+   * to compute the cutoff by itself using the lifetime value. */
+  cache_clean_v3_as_client(now);
+}
+
+/* Purge the client descriptor cache. */
+void
+hs_cache_purge_as_client(void)
+{
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_client, key,
+                              hs_cache_client_descriptor_t *, entry) {
+    size_t entry_size = cache_get_client_entry_size(entry);
+    MAP_DEL_CURRENT(key);
+    cache_client_desc_free(entry);
+    /* Update our OOM. We didn't use the remove() function because we are in
+     * a loop so we have to explicitly decrement. */
+    rend_cache_decrement_allocation(entry_size);
+  } DIGEST256MAP_FOREACH_END;
+
+  log_info(LD_REND, "Hidden service client descriptor cache purged.");
+}
+
+/* For a given service identity public key and an introduction authentication
+ * key, note the given failure in the client intro state cache. */
+void
+hs_cache_client_intro_state_note(const ed25519_public_key_t *service_pk,
+                                 const ed25519_public_key_t *auth_key,
+                                 rend_intro_point_failure_t failure)
+{
+  int found;
+  hs_cache_intro_state_t *entry;
+
+  tor_assert(service_pk);
+  tor_assert(auth_key);
+
+  found = cache_client_intro_state_lookup(service_pk, auth_key, &entry);
+  if (!found) {
+    /* Create a new entry and add it to the cache. */
+    cache_client_intro_state_add(service_pk, auth_key, &entry);
+  }
+  /* Note down the entry. */
+  cache_client_intro_state_note(entry, failure);
+}
+
+/* For a given service identity public key and an introduction authentication
+ * key, return true iff it is present in the failure cache. */
+const hs_cache_intro_state_t *
+hs_cache_client_intro_state_find(const ed25519_public_key_t *service_pk,
+                                 const ed25519_public_key_t *auth_key)
+{
+  hs_cache_intro_state_t *state = NULL;
+  cache_client_intro_state_lookup(service_pk, auth_key, &state);
+  return state;
+}
+
+/* Cleanup the client introduction state cache. */
+void
+hs_cache_client_intro_state_clean(time_t now)
+{
+  time_t cutoff = now - HS_CACHE_CLIENT_INTRO_STATE_MAX_AGE;
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_client_intro_state, key,
+                              hs_cache_client_intro_state_t *, cache) {
+    /* Cleanup intro points failure. */
+    cache_client_intro_state_clean(cutoff, cache);
+
+    /* Is this cache empty for this service key? If yes, remove it from the
+     * cache. Else keep it. */
+    if (cache_client_intro_state_is_empty(cache)) {
+      cache_client_intro_state_free(cache);
+      MAP_DEL_CURRENT(key);
+    }
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Purge the client introduction state cache. */
+void
+hs_cache_client_intro_state_purge(void)
+{
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_client_intro_state, key,
+                              hs_cache_client_intro_state_t *, cache) {
+    MAP_DEL_CURRENT(key);
+    cache_client_intro_state_free(cache);
+  } DIGEST256MAP_FOREACH_END;
+
+  log_info(LD_REND, "Hidden service client introduction point state "
+                    "cache purged.");
+}
+
+/**************** Generics *********************************/
+
+/* Do a round of OOM cleanup on all directory caches. Return the amount of
+ * removed bytes. It is possible that the returned value is lower than
+ * min_remove_bytes if the caches get emptied out so the caller should be
+ * aware of this. */
+size_t
+hs_cache_handle_oom(time_t now, size_t min_remove_bytes)
+{
+  time_t k;
+  size_t bytes_removed = 0;
+
+  /* Our OOM handler called with 0 bytes to remove is a code flow error. */
+  tor_assert(min_remove_bytes != 0);
+
+  /* The algorithm is as follow. K is the oldest expected descriptor age.
+   *
+   *   1) Deallocate all entries from v2 cache that are older than K hours.
+   *      1.1) If the amount of remove bytes has been reached, stop.
+   *   2) Deallocate all entries from v3 cache that are older than K hours
+   *      2.1) If the amount of remove bytes has been reached, stop.
+   *   3) Set K = K - RendPostPeriod and repeat process until K is < 0.
+   *
+   * This ends up being O(Kn).
+   */
+
+  /* Set K to the oldest expected age in seconds which is the maximum
+   * lifetime of a cache entry. We'll use the v2 lifetime because it's much
+   * bigger than the v3 thus leading to cleaning older descriptors. */
+  k = rend_cache_max_entry_lifetime();
+
+  do {
+    time_t cutoff;
+
+    /* If K becomes negative, it means we've empty the caches so stop and
+     * return what we were able to cleanup. */
+    if (k < 0) {
+      break;
+    }
+    /* Compute a cutoff value with K and the current time. */
+    cutoff = now - k;
+
+    /* Start by cleaning the v2 cache with that cutoff. */
+    bytes_removed += rend_cache_clean_v2_descs_as_dir(cutoff);
+
+    if (bytes_removed < min_remove_bytes) {
+      /* We haven't remove enough bytes so clean v3 cache. */
+      bytes_removed += cache_clean_v3_as_dir(now, cutoff);
+      /* Decrement K by a post period to shorten the cutoff. */
+      k -= get_options()->RendPostPeriod;
+    }
+  } while (bytes_removed < min_remove_bytes);
+
+  return bytes_removed;
+}
+
+/* Return the maximum size of a v3 HS descriptor. */
+unsigned int
+hs_cache_get_max_descriptor_size(void)
+{
+  return (unsigned) networkstatus_get_param(NULL,
+                                            "HSV3MaxDescriptorSize",
+                                            HS_DESC_MAX_LEN, 1, INT32_MAX);
+}
+
+/* Initialize the hidden service cache subsystem. */
+void
+hs_cache_init(void)
+{
+  /* Calling this twice is very wrong code flow. */
+  tor_assert(!hs_cache_v3_dir);
+  hs_cache_v3_dir = digest256map_new();
+
+  tor_assert(!hs_cache_v3_client);
+  hs_cache_v3_client = digest256map_new();
+
+  tor_assert(!hs_cache_client_intro_state);
+  hs_cache_client_intro_state = digest256map_new();
+}
+
+/* Cleanup the hidden service cache subsystem. */
+void
+hs_cache_free_all(void)
+{
+  digest256map_free(hs_cache_v3_dir, cache_dir_desc_free_void);
+  hs_cache_v3_dir = NULL;
+
+  digest256map_free(hs_cache_v3_client, cache_client_desc_free_void);
+  hs_cache_v3_client = NULL;
+
+  digest256map_free(hs_cache_client_intro_state,
+                    cache_client_intro_state_free_void);
+  hs_cache_client_intro_state = NULL;
+}
diff --git a/src/feature/hs/hs_cache.h b/src/feature/hs/hs_cache.h
new file mode 100644
index 0000000000..1e479700fa
--- /dev/null
+++ b/src/feature/hs/hs_cache.h
@@ -0,0 +1,130 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cache.h
+ * \brief Header file for hs_cache.c
+ **/
+
+#ifndef TOR_HS_CACHE_H
+#define TOR_HS_CACHE_H
+
+#include <stdint.h>
+
+#include "or/hs_common.h"
+#include "or/hs_descriptor.h"
+#include "or/rendcommon.h"
+#include "or/torcert.h"
+
+struct ed25519_public_key_t;
+
+/* This is the maximum time an introduction point state object can stay in the
+ * client cache in seconds (2 mins or 120 seconds). */
+#define HS_CACHE_CLIENT_INTRO_STATE_MAX_AGE (2 * 60)
+
+/* Introduction point state. */
+typedef struct hs_cache_intro_state_t {
+  /* When this entry was created and put in the cache. */
+  time_t created_ts;
+
+  /* Did it suffered a generic error? */
+  unsigned int error : 1;
+
+  /* Did it timed out? */
+  unsigned int timed_out : 1;
+
+  /* How many times we tried to reached it and it was unreachable. */
+  uint32_t unreachable_count;
+} hs_cache_intro_state_t;
+
+typedef struct hs_cache_client_intro_state_t {
+  /* Contains hs_cache_intro_state_t object indexed by introduction point
+   * authentication key. */
+  digest256map_t *intro_points;
+} hs_cache_client_intro_state_t;
+
+/* Descriptor representation on the directory side which is a subset of
+ * information that the HSDir can decode and serve it. */
+typedef struct hs_cache_dir_descriptor_t {
+  /* This object is indexed using the blinded pubkey located in the plaintext
+   * data which is populated only once the descriptor has been successfully
+   * decoded and validated. This simply points to that pubkey. */
+  const uint8_t *key;
+
+  /* When does this entry has been created. Used to expire entries. */
+  time_t created_ts;
+
+  /* Descriptor plaintext information. Obviously, we can't decrypt the
+   * encrypted part of the descriptor. */
+  hs_desc_plaintext_data_t *plaintext_data;
+
+  /* Encoded descriptor which is basically in text form. It's a NUL terminated
+   * string thus safe to strlen(). */
+  char *encoded_desc;
+} hs_cache_dir_descriptor_t;
+
+/* Public API */
+
+void hs_cache_init(void);
+void hs_cache_free_all(void);
+void hs_cache_clean_as_dir(time_t now);
+size_t hs_cache_handle_oom(time_t now, size_t min_remove_bytes);
+
+unsigned int hs_cache_get_max_descriptor_size(void);
+
+/* Store and Lookup function. They are version agnostic that is depending on
+ * the requested version of the descriptor, it will be re-routed to the
+ * right function. */
+int hs_cache_store_as_dir(const char *desc);
+int hs_cache_lookup_as_dir(uint32_t version, const char *query,
+                           const char **desc_out);
+
+const hs_descriptor_t *
+hs_cache_lookup_as_client(const struct ed25519_public_key_t *key);
+const char *
+hs_cache_lookup_encoded_as_client(const struct ed25519_public_key_t *key);
+int hs_cache_store_as_client(const char *desc_str,
+                             const struct ed25519_public_key_t *identity_pk);
+void hs_cache_clean_as_client(time_t now);
+void hs_cache_purge_as_client(void);
+
+/* Client failure cache. */
+void hs_cache_client_intro_state_note(
+                              const struct ed25519_public_key_t *service_pk,
+                              const struct ed25519_public_key_t *auth_key,
+                              rend_intro_point_failure_t failure);
+const hs_cache_intro_state_t *hs_cache_client_intro_state_find(
+                              const struct ed25519_public_key_t *service_pk,
+                              const struct ed25519_public_key_t *auth_key);
+void hs_cache_client_intro_state_clean(time_t now);
+void hs_cache_client_intro_state_purge(void);
+
+#ifdef HS_CACHE_PRIVATE
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/** Represents a locally cached HS descriptor on a hidden service client. */
+typedef struct hs_cache_client_descriptor_t {
+  /* This object is indexed using the service identity public key */
+  struct ed25519_public_key_t key;
+
+  /* When will this entry expire? We expire cached client descriptors in the
+   * start of the next time period, since that's when clients need to start
+   * using the next blinded key of the service. */
+  time_t expiration_ts;
+
+  /* The cached descriptor, this object is the owner. It can't be NULL. A
+   * cache object without a valid descriptor is not possible. */
+  hs_descriptor_t *desc;
+
+  /* Encoded descriptor in string form. Can't be NULL. */
+  char *encoded_desc;
+} hs_cache_client_descriptor_t;
+
+STATIC size_t cache_clean_v3_as_dir(time_t now, time_t global_cutoff);
+
+STATIC hs_cache_client_descriptor_t *
+lookup_v3_desc_as_client(const uint8_t *key);
+
+#endif /* defined(HS_CACHE_PRIVATE) */
+
+#endif /* !defined(TOR_HS_CACHE_H) */
diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c
new file mode 100644
index 0000000000..f8b76c5133
--- /dev/null
+++ b/src/feature/hs/hs_cell.c
@@ -0,0 +1,950 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cell.c
+ * \brief Hidden service API for cell creation and handling.
+ **/
+
+#include "or/or.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/rendservice.h"
+#include "or/replaycache.h"
+
+#include "or/hs_cell.h"
+#include "or/hs_ntor.h"
+
+#include "or/origin_circuit_st.h"
+
+/* Trunnel. */
+#include "trunnel/ed25519_cert.h"
+#include "trunnel/hs/cell_common.h"
+#include "trunnel/hs/cell_establish_intro.h"
+#include "trunnel/hs/cell_introduce1.h"
+#include "trunnel/hs/cell_rendezvous.h"
+
+/* Compute the MAC of an INTRODUCE cell in mac_out. The encoded_cell param is
+ * the cell content up to the ENCRYPTED section of length encoded_cell_len.
+ * The encrypted param is the start of the ENCRYPTED section of length
+ * encrypted_len. The mac_key is the key needed for the computation of the MAC
+ * derived from the ntor handshake of length mac_key_len.
+ *
+ * The length mac_out_len must be at least DIGEST256_LEN. */
+static void
+compute_introduce_mac(const uint8_t *encoded_cell, size_t encoded_cell_len,
+                      const uint8_t *encrypted, size_t encrypted_len,
+                      const uint8_t *mac_key, size_t mac_key_len,
+                      uint8_t *mac_out, size_t mac_out_len)
+{
+  size_t offset = 0;
+  size_t mac_msg_len;
+  uint8_t mac_msg[RELAY_PAYLOAD_SIZE] = {0};
+
+  tor_assert(encoded_cell);
+  tor_assert(encrypted);
+  tor_assert(mac_key);
+  tor_assert(mac_out);
+  tor_assert(mac_out_len >= DIGEST256_LEN);
+
+  /* Compute the size of the message which is basically the entire cell until
+   * the MAC field of course. */
+  mac_msg_len = encoded_cell_len + (encrypted_len - DIGEST256_LEN);
+  tor_assert(mac_msg_len <= sizeof(mac_msg));
+
+  /* First, put the encoded cell in the msg. */
+  memcpy(mac_msg, encoded_cell, encoded_cell_len);
+  offset += encoded_cell_len;
+  /* Second, put the CLIENT_PK + ENCRYPTED_DATA but ommit the MAC field (which
+   * is junk at this point). */
+  memcpy(mac_msg + offset, encrypted, (encrypted_len - DIGEST256_LEN));
+  offset += (encrypted_len - DIGEST256_LEN);
+  tor_assert(offset == mac_msg_len);
+
+  crypto_mac_sha3_256(mac_out, mac_out_len,
+                      mac_key, mac_key_len,
+                      mac_msg, mac_msg_len);
+  memwipe(mac_msg, 0, sizeof(mac_msg));
+}
+
+/* From a set of keys, subcredential and the ENCRYPTED section of an
+ * INTRODUCE2 cell, return a newly allocated intro cell keys structure.
+ * Finally, the client public key is copied in client_pk. On error, return
+ * NULL. */
+static hs_ntor_intro_cell_keys_t *
+get_introduce2_key_material(const ed25519_public_key_t *auth_key,
+                            const curve25519_keypair_t *enc_key,
+                            const uint8_t *subcredential,
+                            const uint8_t *encrypted_section,
+                            curve25519_public_key_t *client_pk)
+{
+  hs_ntor_intro_cell_keys_t *keys;
+
+  tor_assert(auth_key);
+  tor_assert(enc_key);
+  tor_assert(subcredential);
+  tor_assert(encrypted_section);
+  tor_assert(client_pk);
+
+  keys = tor_malloc_zero(sizeof(*keys));
+
+  /* First bytes of the ENCRYPTED section are the client public key. */
+  memcpy(client_pk->public_key, encrypted_section, CURVE25519_PUBKEY_LEN);
+
+  if (hs_ntor_service_get_introduce1_keys(auth_key, enc_key, client_pk,
+                                          subcredential, keys) < 0) {
+    /* Don't rely on the caller to wipe this on error. */
+    memwipe(client_pk, 0, sizeof(curve25519_public_key_t));
+    tor_free(keys);
+    keys = NULL;
+  }
+  return keys;
+}
+
+/* Using the given encryption key, decrypt the encrypted_section of length
+ * encrypted_section_len of an INTRODUCE2 cell and return a newly allocated
+ * buffer containing the decrypted data. On decryption failure, NULL is
+ * returned. */
+static uint8_t *
+decrypt_introduce2(const uint8_t *enc_key, const uint8_t *encrypted_section,
+                   size_t encrypted_section_len)
+{
+  uint8_t *decrypted = NULL;
+  crypto_cipher_t *cipher = NULL;
+
+  tor_assert(enc_key);
+  tor_assert(encrypted_section);
+
+  /* Decrypt ENCRYPTED section. */
+  cipher = crypto_cipher_new_with_bits((char *) enc_key,
+                                       CURVE25519_PUBKEY_LEN * 8);
+  tor_assert(cipher);
+
+  /* This is symmetric encryption so can't be bigger than the encrypted
+   * section length. */
+  decrypted = tor_malloc_zero(encrypted_section_len);
+  if (crypto_cipher_decrypt(cipher, (char *) decrypted,
+                            (const char *) encrypted_section,
+                            encrypted_section_len) < 0) {
+    tor_free(decrypted);
+    decrypted = NULL;
+    goto done;
+  }
+
+ done:
+  crypto_cipher_free(cipher);
+  return decrypted;
+}
+
+/* Given a pointer to the decrypted data of the ENCRYPTED section of an
+ * INTRODUCE2 cell of length decrypted_len, parse and validate the cell
+ * content. Return a newly allocated cell structure or NULL on error. The
+ * circuit and service object are only used for logging purposes. */
+static trn_cell_introduce_encrypted_t *
+parse_introduce2_encrypted(const uint8_t *decrypted_data,
+                           size_t decrypted_len, const origin_circuit_t *circ,
+                           const hs_service_t *service)
+{
+  trn_cell_introduce_encrypted_t *enc_cell = NULL;
+
+  tor_assert(decrypted_data);
+  tor_assert(circ);
+  tor_assert(service);
+
+  if (trn_cell_introduce_encrypted_parse(&enc_cell, decrypted_data,
+                                         decrypted_len) < 0) {
+    log_info(LD_REND, "Unable to parse the decrypted ENCRYPTED section of "
+                      "the INTRODUCE2 cell on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  if (trn_cell_introduce_encrypted_get_onion_key_type(enc_cell) !=
+      HS_CELL_ONION_KEY_TYPE_NTOR) {
+    log_info(LD_REND, "INTRODUCE2 onion key type is invalid. Got %u but "
+                      "expected %u on circuit %u for service %s",
+             trn_cell_introduce_encrypted_get_onion_key_type(enc_cell),
+             HS_CELL_ONION_KEY_TYPE_NTOR, TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  if (trn_cell_introduce_encrypted_getlen_onion_key(enc_cell) !=
+      CURVE25519_PUBKEY_LEN) {
+    log_info(LD_REND, "INTRODUCE2 onion key length is invalid. Got %u but "
+                      "expected %d on circuit %u for service %s",
+             (unsigned)trn_cell_introduce_encrypted_getlen_onion_key(enc_cell),
+             CURVE25519_PUBKEY_LEN, TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+  /* XXX: Validate NSPEC field as well. */
+
+  return enc_cell;
+ err:
+  trn_cell_introduce_encrypted_free(enc_cell);
+  return NULL;
+}
+
+/* Build a legacy ESTABLISH_INTRO cell with the given circuit nonce and RSA
+ * encryption key. The encoded cell is put in cell_out that MUST at least be
+ * of the size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on
+ * success else a negative value and cell_out is untouched. */
+static ssize_t
+build_legacy_establish_intro(const char *circ_nonce, crypto_pk_t *enc_key,
+                             uint8_t *cell_out)
+{
+  ssize_t cell_len;
+
+  tor_assert(circ_nonce);
+  tor_assert(enc_key);
+  tor_assert(cell_out);
+
+  memwipe(cell_out, 0, RELAY_PAYLOAD_SIZE);
+
+  cell_len = rend_service_encode_establish_intro_cell((char*)cell_out,
+                                                      RELAY_PAYLOAD_SIZE,
+                                                      enc_key, circ_nonce);
+  return cell_len;
+}
+
+/* Parse an INTRODUCE2 cell from payload of size payload_len for the given
+ * service and circuit which are used only for logging purposes. The resulting
+ * parsed cell is put in cell_ptr_out.
+ *
+ * This function only parses prop224 INTRODUCE2 cells even when the intro point
+ * is a legacy intro point. That's because intro points don't actually care
+ * about the contents of the introduce cell. Legacy INTRODUCE cells are only
+ * used by the legacy system now.
+ *
+ * Return 0 on success else a negative value and cell_ptr_out is untouched. */
+static int
+parse_introduce2_cell(const hs_service_t *service,
+                      const origin_circuit_t *circ, const uint8_t *payload,
+                      size_t payload_len,
+                      trn_cell_introduce1_t **cell_ptr_out)
+{
+  trn_cell_introduce1_t *cell = NULL;
+
+  tor_assert(service);
+  tor_assert(circ);
+  tor_assert(payload);
+  tor_assert(cell_ptr_out);
+
+  /* Parse the cell so we can start cell validation. */
+  if (trn_cell_introduce1_parse(&cell, payload, payload_len) < 0) {
+    log_info(LD_PROTOCOL, "Unable to parse INTRODUCE2 cell on circuit %u "
+                          "for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  /* Success. */
+  *cell_ptr_out = cell;
+  return 0;
+ err:
+  return -1;
+}
+
+/* Set the onion public key onion_pk in cell, the encrypted section of an
+ * INTRODUCE1 cell. */
+static void
+introduce1_set_encrypted_onion_key(trn_cell_introduce_encrypted_t *cell,
+                                   const uint8_t *onion_pk)
+{
+  tor_assert(cell);
+  tor_assert(onion_pk);
+  /* There is only one possible key type for a non legacy cell. */
+  trn_cell_introduce_encrypted_set_onion_key_type(cell,
+                                                  HS_CELL_ONION_KEY_TYPE_NTOR);
+  trn_cell_introduce_encrypted_set_onion_key_len(cell, CURVE25519_PUBKEY_LEN);
+  trn_cell_introduce_encrypted_setlen_onion_key(cell, CURVE25519_PUBKEY_LEN);
+  memcpy(trn_cell_introduce_encrypted_getarray_onion_key(cell), onion_pk,
+         trn_cell_introduce_encrypted_getlen_onion_key(cell));
+}
+
+/* Set the link specifiers in lspecs in cell, the encrypted section of an
+ * INTRODUCE1 cell. */
+static void
+introduce1_set_encrypted_link_spec(trn_cell_introduce_encrypted_t *cell,
+                                   const smartlist_t *lspecs)
+{
+  tor_assert(cell);
+  tor_assert(lspecs);
+  tor_assert(smartlist_len(lspecs) > 0);
+  tor_assert(smartlist_len(lspecs) <= UINT8_MAX);
+
+  uint8_t lspecs_num = (uint8_t) smartlist_len(lspecs);
+  trn_cell_introduce_encrypted_set_nspec(cell, lspecs_num);
+  /* We aren't duplicating the link specifiers object here which means that
+   * the ownership goes to the trn_cell_introduce_encrypted_t cell and those
+   * object will be freed when the cell is. */
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls,
+                    trn_cell_introduce_encrypted_add_nspecs(cell, ls));
+}
+
+/* Set padding in the enc_cell only if needed that is the total length of both
+ * sections are below the mininum required for an INTRODUCE1 cell. */
+static void
+introduce1_set_encrypted_padding(const trn_cell_introduce1_t *cell,
+                                 trn_cell_introduce_encrypted_t *enc_cell)
+{
+  tor_assert(cell);
+  tor_assert(enc_cell);
+  /* This is the length we expect to have once encoded of the whole cell. */
+  ssize_t full_len = trn_cell_introduce1_encoded_len(cell) +
+                     trn_cell_introduce_encrypted_encoded_len(enc_cell);
+  tor_assert(full_len > 0);
+  if (full_len < HS_CELL_INTRODUCE1_MIN_SIZE) {
+    size_t padding = HS_CELL_INTRODUCE1_MIN_SIZE - full_len;
+    trn_cell_introduce_encrypted_setlen_pad(enc_cell, padding);
+    memset(trn_cell_introduce_encrypted_getarray_pad(enc_cell), 0,
+           trn_cell_introduce_encrypted_getlen_pad(enc_cell));
+  }
+}
+
+/* Encrypt the ENCRYPTED payload and encode it in the cell using the enc_cell
+ * and the INTRODUCE1 data.
+ *
+ * This can't fail but it is very important that the caller sets every field
+ * in data so the computation of the INTRODUCE1 keys doesn't fail. */
+static void
+introduce1_encrypt_and_encode(trn_cell_introduce1_t *cell,
+                              const trn_cell_introduce_encrypted_t *enc_cell,
+                              const hs_cell_introduce1_data_t *data)
+{
+  size_t offset = 0;
+  ssize_t encrypted_len;
+  ssize_t encoded_cell_len, encoded_enc_cell_len;
+  uint8_t encoded_cell[RELAY_PAYLOAD_SIZE] = {0};
+  uint8_t encoded_enc_cell[RELAY_PAYLOAD_SIZE] = {0};
+  uint8_t *encrypted = NULL;
+  uint8_t mac[DIGEST256_LEN];
+  crypto_cipher_t *cipher = NULL;
+  hs_ntor_intro_cell_keys_t keys;
+
+  tor_assert(cell);
+  tor_assert(enc_cell);
+  tor_assert(data);
+
+  /* Encode the cells up to now of what we have to we can perform the MAC
+   * computation on it. */
+  encoded_cell_len = trn_cell_introduce1_encode(encoded_cell,
+                                                sizeof(encoded_cell), cell);
+  /* We have a much more serious issue if this isn't true. */
+  tor_assert(encoded_cell_len > 0);
+
+  encoded_enc_cell_len =
+    trn_cell_introduce_encrypted_encode(encoded_enc_cell,
+                                        sizeof(encoded_enc_cell), enc_cell);
+  /* We have a much more serious issue if this isn't true. */
+  tor_assert(encoded_enc_cell_len > 0);
+
+  /* Get the key material for the encryption. */
+  if (hs_ntor_client_get_introduce1_keys(data->auth_pk, data->enc_pk,
+                                         data->client_kp,
+                                         data->subcredential, &keys) < 0) {
+    tor_assert_unreached();
+  }
+
+  /* Prepare cipher with the encryption key just computed. */
+  cipher = crypto_cipher_new_with_bits((const char *) keys.enc_key,
+                                       sizeof(keys.enc_key) * 8);
+  tor_assert(cipher);
+
+  /* Compute the length of the ENCRYPTED section which is the CLIENT_PK,
+   * ENCRYPTED_DATA and MAC length. */
+  encrypted_len = sizeof(data->client_kp->pubkey) + encoded_enc_cell_len +
+                  sizeof(mac);
+  tor_assert(encrypted_len < RELAY_PAYLOAD_SIZE);
+  encrypted = tor_malloc_zero(encrypted_len);
+
+  /* Put the CLIENT_PK first. */
+  memcpy(encrypted, data->client_kp->pubkey.public_key,
+         sizeof(data->client_kp->pubkey.public_key));
+  offset += sizeof(data->client_kp->pubkey.public_key);
+  /* Then encrypt and set the ENCRYPTED_DATA. This can't fail. */
+  crypto_cipher_encrypt(cipher, (char *) encrypted + offset,
+                        (const char *) encoded_enc_cell, encoded_enc_cell_len);
+  crypto_cipher_free(cipher);
+  offset += encoded_enc_cell_len;
+  /* Compute MAC from the above and put it in the buffer. This function will
+   * make the adjustment to the encrypted_len to omit the MAC length. */
+  compute_introduce_mac(encoded_cell, encoded_cell_len,
+                        encrypted, encrypted_len,
+                        keys.mac_key, sizeof(keys.mac_key),
+                        mac, sizeof(mac));
+  memcpy(encrypted + offset, mac, sizeof(mac));
+  offset += sizeof(mac);
+  tor_assert(offset == (size_t) encrypted_len);
+
+  /* Set the ENCRYPTED section in the cell. */
+  trn_cell_introduce1_setlen_encrypted(cell, encrypted_len);
+  memcpy(trn_cell_introduce1_getarray_encrypted(cell),
+         encrypted, encrypted_len);
+
+  /* Cleanup. */
+  memwipe(&keys, 0, sizeof(keys));
+  memwipe(mac, 0, sizeof(mac));
+  memwipe(encrypted, 0, sizeof(encrypted_len));
+  memwipe(encoded_enc_cell, 0, sizeof(encoded_enc_cell));
+  tor_free(encrypted);
+}
+
+/* Using the INTRODUCE1 data, setup the ENCRYPTED section in cell. This means
+ * set it, encrypt it and encode it. */
+static void
+introduce1_set_encrypted(trn_cell_introduce1_t *cell,
+                         const hs_cell_introduce1_data_t *data)
+{
+  trn_cell_introduce_encrypted_t *enc_cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(cell);
+  tor_assert(data);
+
+  enc_cell = trn_cell_introduce_encrypted_new();
+  tor_assert(enc_cell);
+
+  /* Set extension data. None are used. */
+  ext = trn_cell_extension_new();
+  tor_assert(ext);
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce_encrypted_set_extensions(enc_cell, ext);
+
+  /* Set the rendezvous cookie. */
+  memcpy(trn_cell_introduce_encrypted_getarray_rend_cookie(enc_cell),
+         data->rendezvous_cookie, REND_COOKIE_LEN);
+
+  /* Set the onion public key. */
+  introduce1_set_encrypted_onion_key(enc_cell, data->onion_pk->public_key);
+
+  /* Set the link specifiers. */
+  introduce1_set_encrypted_link_spec(enc_cell, data->link_specifiers);
+
+  /* Set padding. */
+  introduce1_set_encrypted_padding(cell, enc_cell);
+
+  /* Encrypt and encode it in the cell. */
+  introduce1_encrypt_and_encode(cell, enc_cell, data);
+
+  /* Cleanup. */
+  trn_cell_introduce_encrypted_free(enc_cell);
+}
+
+/* Set the authentication key in the INTRODUCE1 cell from the given data. */
+static void
+introduce1_set_auth_key(trn_cell_introduce1_t *cell,
+                        const hs_cell_introduce1_data_t *data)
+{
+  tor_assert(cell);
+  tor_assert(data);
+  /* There is only one possible type for a non legacy cell. */
+  trn_cell_introduce1_set_auth_key_type(cell, HS_INTRO_AUTH_KEY_TYPE_ED25519);
+  trn_cell_introduce1_set_auth_key_len(cell, ED25519_PUBKEY_LEN);
+  trn_cell_introduce1_setlen_auth_key(cell, ED25519_PUBKEY_LEN);
+  memcpy(trn_cell_introduce1_getarray_auth_key(cell),
+         data->auth_pk->pubkey, trn_cell_introduce1_getlen_auth_key(cell));
+}
+
+/* Set the legacy ID field in the INTRODUCE1 cell from the given data. */
+static void
+introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
+                         const hs_cell_introduce1_data_t *data)
+{
+  tor_assert(cell);
+  tor_assert(data);
+
+  if (data->is_legacy) {
+    uint8_t digest[DIGEST_LEN];
+    if (BUG(crypto_pk_get_digest(data->legacy_key, (char *) digest) < 0)) {
+      return;
+    }
+    memcpy(trn_cell_introduce1_getarray_legacy_key_id(cell),
+           digest, trn_cell_introduce1_getlen_legacy_key_id(cell));
+  } else {
+    /* We have to zeroed the LEGACY_KEY_ID field. */
+    memset(trn_cell_introduce1_getarray_legacy_key_id(cell), 0,
+           trn_cell_introduce1_getlen_legacy_key_id(cell));
+  }
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/* Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point
+ * object. The encoded cell is put in cell_out that MUST at least be of the
+ * size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on success else
+ * a negative value and cell_out is untouched. This function also supports
+ * legacy cell creation. */
+ssize_t
+hs_cell_build_establish_intro(const char *circ_nonce,
+                              const hs_service_intro_point_t *ip,
+                              uint8_t *cell_out)
+{
+  ssize_t cell_len = -1;
+  uint16_t sig_len = ED25519_SIG_LEN;
+  trn_cell_extension_t *ext;
+  trn_cell_establish_intro_t *cell = NULL;
+
+  tor_assert(circ_nonce);
+  tor_assert(ip);
+
+  /* Quickly handle the legacy IP. */
+  if (ip->base.is_only_legacy) {
+    tor_assert(ip->legacy_key);
+    cell_len = build_legacy_establish_intro(circ_nonce, ip->legacy_key,
+                                            cell_out);
+    tor_assert(cell_len <= RELAY_PAYLOAD_SIZE);
+    /* Success or not we are done here. */
+    goto done;
+  }
+
+  /* Set extension data. None used here. */
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  cell = trn_cell_establish_intro_new();
+  trn_cell_establish_intro_set_extensions(cell, ext);
+  /* Set signature size. Array is then allocated in the cell. We need to do
+   * this early so we can use trunnel API to get the signature length. */
+  trn_cell_establish_intro_set_sig_len(cell, sig_len);
+  trn_cell_establish_intro_setlen_sig(cell, sig_len);
+
+  /* Set AUTH_KEY_TYPE: 2 means ed25519 */
+  trn_cell_establish_intro_set_auth_key_type(cell,
+                                             HS_INTRO_AUTH_KEY_TYPE_ED25519);
+
+  /* Set AUTH_KEY and AUTH_KEY_LEN field. Must also set byte-length of
+   * AUTH_KEY to match */
+  {
+    uint16_t auth_key_len = ED25519_PUBKEY_LEN;
+    trn_cell_establish_intro_set_auth_key_len(cell, auth_key_len);
+    trn_cell_establish_intro_setlen_auth_key(cell, auth_key_len);
+    /* We do this call _after_ setting the length because it's reallocated at
+     * that point only. */
+    uint8_t *auth_key_ptr = trn_cell_establish_intro_getarray_auth_key(cell);
+    memcpy(auth_key_ptr, ip->auth_key_kp.pubkey.pubkey, auth_key_len);
+  }
+
+  /* Calculate HANDSHAKE_AUTH field (MAC). */
+  {
+    ssize_t tmp_cell_enc_len = 0;
+    ssize_t tmp_cell_mac_offset =
+      sig_len + sizeof(cell->sig_len) +
+      trn_cell_establish_intro_getlen_handshake_mac(cell);
+    uint8_t tmp_cell_enc[RELAY_PAYLOAD_SIZE] = {0};
+    uint8_t mac[TRUNNEL_SHA3_256_LEN], *handshake_ptr;
+
+    /* We first encode the current fields we have in the cell so we can
+     * compute the MAC using the raw bytes. */
+    tmp_cell_enc_len = trn_cell_establish_intro_encode(tmp_cell_enc,
+                                                       sizeof(tmp_cell_enc),
+                                                       cell);
+    if (BUG(tmp_cell_enc_len < 0)) {
+      goto done;
+    }
+    /* Sanity check. */
+    tor_assert(tmp_cell_enc_len > tmp_cell_mac_offset);
+
+    /* Circuit nonce is always DIGEST_LEN according to tor-spec.txt. */
+    crypto_mac_sha3_256(mac, sizeof(mac),
+                        (uint8_t *) circ_nonce, DIGEST_LEN,
+                        tmp_cell_enc, tmp_cell_enc_len - tmp_cell_mac_offset);
+    handshake_ptr = trn_cell_establish_intro_getarray_handshake_mac(cell);
+    memcpy(handshake_ptr, mac, sizeof(mac));
+
+    memwipe(mac, 0, sizeof(mac));
+    memwipe(tmp_cell_enc, 0, sizeof(tmp_cell_enc));
+  }
+
+  /* Calculate the cell signature SIG. */
+  {
+    ssize_t tmp_cell_enc_len = 0;
+    ssize_t tmp_cell_sig_offset = (sig_len + sizeof(cell->sig_len));
+    uint8_t tmp_cell_enc[RELAY_PAYLOAD_SIZE] = {0}, *sig_ptr;
+    ed25519_signature_t sig;
+
+    /* We first encode the current fields we have in the cell so we can
+     * compute the signature from the raw bytes of the cell. */
+    tmp_cell_enc_len = trn_cell_establish_intro_encode(tmp_cell_enc,
+                                                       sizeof(tmp_cell_enc),
+                                                       cell);
+    if (BUG(tmp_cell_enc_len < 0)) {
+      goto done;
+    }
+
+    if (ed25519_sign_prefixed(&sig, tmp_cell_enc,
+                              tmp_cell_enc_len - tmp_cell_sig_offset,
+                              ESTABLISH_INTRO_SIG_PREFIX, &ip->auth_key_kp)) {
+      log_warn(LD_BUG, "Unable to make signature for ESTABLISH_INTRO cell.");
+      goto done;
+    }
+    /* Copy the signature into the cell. */
+    sig_ptr = trn_cell_establish_intro_getarray_sig(cell);
+    memcpy(sig_ptr, sig.sig, sig_len);
+
+    memwipe(tmp_cell_enc, 0, sizeof(tmp_cell_enc));
+  }
+
+  /* Encode the cell. Can't be bigger than a standard cell. */
+  cell_len = trn_cell_establish_intro_encode(cell_out, RELAY_PAYLOAD_SIZE,
+                                             cell);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  return cell_len;
+}
+
+/* Parse the INTRO_ESTABLISHED cell in the payload of size payload_len. If we
+ * are successful at parsing it, return the length of the parsed cell else a
+ * negative value on error. */
+ssize_t
+hs_cell_parse_intro_established(const uint8_t *payload, size_t payload_len)
+{
+  ssize_t ret;
+  trn_cell_intro_established_t *cell = NULL;
+
+  tor_assert(payload);
+
+  /* Try to parse the payload into a cell making sure we do actually have a
+   * valid cell. */
+  ret = trn_cell_intro_established_parse(&cell, payload, payload_len);
+  if (ret >= 0) {
+    /* On success, we do not keep the cell, we just notify the caller that it
+     * was successfully parsed. */
+    trn_cell_intro_established_free(cell);
+  }
+  return ret;
+}
+
+/* Parsse the INTRODUCE2 cell using data which contains everything we need to
+ * do so and contains the destination buffers of information we extract and
+ * compute from the cell. Return 0 on success else a negative value. The
+ * service and circ are only used for logging purposes. */
+ssize_t
+hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
+                         const origin_circuit_t *circ,
+                         const hs_service_t *service)
+{
+  int ret = -1;
+  time_t elapsed;
+  uint8_t *decrypted = NULL;
+  size_t encrypted_section_len;
+  const uint8_t *encrypted_section;
+  trn_cell_introduce1_t *cell = NULL;
+  trn_cell_introduce_encrypted_t *enc_cell = NULL;
+  hs_ntor_intro_cell_keys_t *intro_keys = NULL;
+
+  tor_assert(data);
+  tor_assert(circ);
+  tor_assert(service);
+
+  /* Parse the cell into a decoded data structure pointed by cell_ptr. */
+  if (parse_introduce2_cell(service, circ, data->payload, data->payload_len,
+                            &cell) < 0) {
+    goto done;
+  }
+
+  log_info(LD_REND, "Received a decodable INTRODUCE2 cell on circuit %u "
+                    "for service %s. Decoding encrypted section...",
+           TO_CIRCUIT(circ)->n_circ_id,
+           safe_str_client(service->onion_address));
+
+  encrypted_section = trn_cell_introduce1_getconstarray_encrypted(cell);
+  encrypted_section_len = trn_cell_introduce1_getlen_encrypted(cell);
+
+  /* Encrypted section must at least contain the CLIENT_PK and MAC which is
+   * defined in section 3.3.2 of the specification. */
+  if (encrypted_section_len < (CURVE25519_PUBKEY_LEN + DIGEST256_LEN)) {
+    log_info(LD_REND, "Invalid INTRODUCE2 encrypted section length "
+                      "for service %s. Dropping cell.",
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Check our replay cache for this introduction point. */
+  if (replaycache_add_test_and_elapsed(data->replay_cache, encrypted_section,
+                                       encrypted_section_len, &elapsed)) {
+    log_warn(LD_REND, "Possible replay detected! An INTRODUCE2 cell with the"
+                      "same ENCRYPTED section was seen %ld seconds ago. "
+                      "Dropping cell.", (long int) elapsed);
+    goto done;
+  }
+
+  /* Build the key material out of the key material found in the cell. */
+  intro_keys = get_introduce2_key_material(data->auth_pk, data->enc_kp,
+                                           data->subcredential,
+                                           encrypted_section,
+                                           &data->client_pk);
+  if (intro_keys == NULL) {
+    log_info(LD_REND, "Invalid INTRODUCE2 encrypted data. Unable to "
+                      "compute key material on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Validate MAC from the cell and our computed key material. The MAC field
+   * in the cell is at the end of the encrypted section. */
+  {
+    uint8_t mac[DIGEST256_LEN];
+    /* The MAC field is at the very end of the ENCRYPTED section. */
+    size_t mac_offset = encrypted_section_len - sizeof(mac);
+    /* Compute the MAC. Use the entire encoded payload with a length up to the
+     * ENCRYPTED section. */
+    compute_introduce_mac(data->payload,
+                          data->payload_len - encrypted_section_len,
+                          encrypted_section, encrypted_section_len,
+                          intro_keys->mac_key, sizeof(intro_keys->mac_key),
+                          mac, sizeof(mac));
+    if (tor_memcmp(mac, encrypted_section + mac_offset, sizeof(mac))) {
+      log_info(LD_REND, "Invalid MAC validation for INTRODUCE2 cell on "
+                        "circuit %u for service %s",
+               TO_CIRCUIT(circ)->n_circ_id,
+               safe_str_client(service->onion_address));
+      goto done;
+    }
+  }
+
+  {
+    /* The ENCRYPTED_DATA section starts just after the CLIENT_PK. */
+    const uint8_t *encrypted_data =
+      encrypted_section + sizeof(data->client_pk);
+    /* It's symmetric encryption so it's correct to use the ENCRYPTED length
+     * for decryption. Computes the length of ENCRYPTED_DATA meaning removing
+     * the CLIENT_PK and MAC length. */
+    size_t encrypted_data_len =
+      encrypted_section_len - (sizeof(data->client_pk) + DIGEST256_LEN);
+
+    /* This decrypts the ENCRYPTED_DATA section of the cell. */
+    decrypted = decrypt_introduce2(intro_keys->enc_key,
+                                   encrypted_data, encrypted_data_len);
+    if (decrypted == NULL) {
+      log_info(LD_REND, "Unable to decrypt the ENCRYPTED section of an "
+                        "INTRODUCE2 cell on circuit %u for service %s",
+               TO_CIRCUIT(circ)->n_circ_id,
+               safe_str_client(service->onion_address));
+      goto done;
+    }
+
+    /* Parse this blob into an encrypted cell structure so we can then extract
+     * the data we need out of it. */
+    enc_cell = parse_introduce2_encrypted(decrypted, encrypted_data_len,
+                                          circ, service);
+    memwipe(decrypted, 0, encrypted_data_len);
+    if (enc_cell == NULL) {
+      goto done;
+    }
+  }
+
+  /* XXX: Implement client authorization checks. */
+
+  /* Extract onion key and rendezvous cookie from the cell used for the
+   * rendezvous point circuit e2e encryption. */
+  memcpy(data->onion_pk.public_key,
+         trn_cell_introduce_encrypted_getconstarray_onion_key(enc_cell),
+         CURVE25519_PUBKEY_LEN);
+  memcpy(data->rendezvous_cookie,
+         trn_cell_introduce_encrypted_getconstarray_rend_cookie(enc_cell),
+         sizeof(data->rendezvous_cookie));
+
+  /* Extract rendezvous link specifiers. */
+  for (size_t idx = 0;
+       idx < trn_cell_introduce_encrypted_get_nspec(enc_cell); idx++) {
+    link_specifier_t *lspec =
+      trn_cell_introduce_encrypted_get_nspecs(enc_cell, idx);
+    smartlist_add(data->link_specifiers, hs_link_specifier_dup(lspec));
+  }
+
+  /* Success. */
+  ret = 0;
+  log_info(LD_REND, "Valid INTRODUCE2 cell. Launching rendezvous circuit.");
+
+ done:
+  if (intro_keys) {
+    memwipe(intro_keys, 0, sizeof(hs_ntor_intro_cell_keys_t));
+    tor_free(intro_keys);
+  }
+  tor_free(decrypted);
+  trn_cell_introduce_encrypted_free(enc_cell);
+  trn_cell_introduce1_free(cell);
+  return ret;
+}
+
+/* Build a RENDEZVOUS1 cell with the given rendezvous cookie and handshake
+ * info. The encoded cell is put in cell_out and the length of the data is
+ * returned. This can't fail. */
+ssize_t
+hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
+                          size_t rendezvous_cookie_len,
+                          const uint8_t *rendezvous_handshake_info,
+                          size_t rendezvous_handshake_info_len,
+                          uint8_t *cell_out)
+{
+  ssize_t cell_len;
+  trn_cell_rendezvous1_t *cell;
+
+  tor_assert(rendezvous_cookie);
+  tor_assert(rendezvous_handshake_info);
+  tor_assert(cell_out);
+
+  cell = trn_cell_rendezvous1_new();
+  /* Set the RENDEZVOUS_COOKIE. */
+  memcpy(trn_cell_rendezvous1_getarray_rendezvous_cookie(cell),
+         rendezvous_cookie, rendezvous_cookie_len);
+  /* Set the HANDSHAKE_INFO. */
+  trn_cell_rendezvous1_setlen_handshake_info(cell,
+                                            rendezvous_handshake_info_len);
+  memcpy(trn_cell_rendezvous1_getarray_handshake_info(cell),
+         rendezvous_handshake_info, rendezvous_handshake_info_len);
+  /* Encoding. */
+  cell_len = trn_cell_rendezvous1_encode(cell_out, RELAY_PAYLOAD_SIZE, cell);
+  tor_assert(cell_len > 0);
+
+  trn_cell_rendezvous1_free(cell);
+  return cell_len;
+}
+
+/* Build an INTRODUCE1 cell from the given data. The encoded cell is put in
+ * cell_out which must be of at least size RELAY_PAYLOAD_SIZE. On success, the
+ * encoded length is returned else a negative value and the content of
+ * cell_out should be ignored. */
+ssize_t
+hs_cell_build_introduce1(const hs_cell_introduce1_data_t *data,
+                         uint8_t *cell_out)
+{
+  ssize_t cell_len;
+  trn_cell_introduce1_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(data);
+  tor_assert(cell_out);
+
+  cell = trn_cell_introduce1_new();
+  tor_assert(cell);
+
+  /* Set extension data. None are used. */
+  ext = trn_cell_extension_new();
+  tor_assert(ext);
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce1_set_extensions(cell, ext);
+
+  /* Set the legacy ID field. */
+  introduce1_set_legacy_id(cell, data);
+
+  /* Set the authentication key. */
+  introduce1_set_auth_key(cell, data);
+
+  /* Set the encrypted section. This will set, encrypt and encode the
+   * ENCRYPTED section in the cell. After this, we'll be ready to encode. */
+  introduce1_set_encrypted(cell, data);
+
+  /* Final encoding. */
+  cell_len = trn_cell_introduce1_encode(cell_out, RELAY_PAYLOAD_SIZE, cell);
+
+  trn_cell_introduce1_free(cell);
+  return cell_len;
+}
+
+/* Build an ESTABLISH_RENDEZVOUS cell from the given rendezvous_cookie. The
+ * encoded cell is put in cell_out which must be of at least
+ * RELAY_PAYLOAD_SIZE. On success, the encoded length is returned and the
+ * caller should clear up the content of the cell.
+ *
+ * This function can't fail. */
+ssize_t
+hs_cell_build_establish_rendezvous(const uint8_t *rendezvous_cookie,
+                                   uint8_t *cell_out)
+{
+  tor_assert(rendezvous_cookie);
+  tor_assert(cell_out);
+
+  memcpy(cell_out, rendezvous_cookie, HS_REND_COOKIE_LEN);
+  return HS_REND_COOKIE_LEN;
+}
+
+/* Handle an INTRODUCE_ACK cell encoded in payload of length payload_len.
+ * Return the status code on success else a negative value if the cell as not
+ * decodable. */
+int
+hs_cell_parse_introduce_ack(const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+  trn_cell_introduce_ack_t *cell = NULL;
+
+  tor_assert(payload);
+
+  /* If it is a legacy IP, rend-spec.txt specifies that a ACK is 0 byte and a
+   * NACK is 1 byte. We can't use the legacy function for this so we have to
+   * do a special case. */
+  if (payload_len <= 1) {
+    if (payload_len == 0) {
+      ret = HS_CELL_INTRO_ACK_SUCCESS;
+    } else {
+      ret = HS_CELL_INTRO_ACK_FAILURE;
+    }
+    goto end;
+  }
+
+  if (trn_cell_introduce_ack_parse(&cell, payload, payload_len) < 0) {
+    log_info(LD_REND, "Invalid INTRODUCE_ACK cell. Unable to parse it.");
+    goto end;
+  }
+
+  ret = trn_cell_introduce_ack_get_status(cell);
+
+ end:
+  trn_cell_introduce_ack_free(cell);
+  return ret;
+}
+
+/* Handle a RENDEZVOUS2 cell encoded in payload of length payload_len. On
+ * success, handshake_info contains the data in the HANDSHAKE_INFO field, and
+ * 0 is returned. On error, a negative value is returned. */
+int
+hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
+                          uint8_t *handshake_info, size_t handshake_info_len)
+{
+  int ret = -1;
+  trn_cell_rendezvous2_t *cell = NULL;
+
+  tor_assert(payload);
+  tor_assert(handshake_info);
+
+  if (trn_cell_rendezvous2_parse(&cell, payload, payload_len) < 0) {
+    log_info(LD_REND, "Invalid RENDEZVOUS2 cell. Unable to parse it.");
+    goto end;
+  }
+
+  /* Static size, we should never have an issue with this else we messed up
+   * our code flow. */
+  tor_assert(trn_cell_rendezvous2_getlen_handshake_info(cell) ==
+             handshake_info_len);
+  memcpy(handshake_info,
+         trn_cell_rendezvous2_getconstarray_handshake_info(cell),
+         handshake_info_len);
+  ret = 0;
+
+ end:
+  trn_cell_rendezvous2_free(cell);
+  return ret;
+}
+
+/* Clear the given INTRODUCE1 data structure data. */
+void
+hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data)
+{
+  if (data == NULL) {
+    return;
+  }
+  /* Object in this list have been moved to the cell object when building it
+   * so they've been freed earlier. We do that in order to avoid duplicating
+   * them leading to more memory and CPU time being used for nothing. */
+  smartlist_free(data->link_specifiers);
+  /* The data object has no ownership of any members. */
+  memwipe(data, 0, sizeof(hs_cell_introduce1_data_t));
+}
+
diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h
new file mode 100644
index 0000000000..4a522810c6
--- /dev/null
+++ b/src/feature/hs/hs_cell.h
@@ -0,0 +1,122 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cell.h
+ * \brief Header file containing cell data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_CELL_H
+#define TOR_HS_CELL_H
+
+#include "or/or.h"
+#include "or/hs_service.h"
+
+/* An INTRODUCE1 cell requires at least this amount of bytes (see section
+ * 3.2.2 of the specification). Below this value, the cell must be padded. */
+#define HS_CELL_INTRODUCE1_MIN_SIZE 246
+
+/* Status code of an INTRODUCE_ACK cell. */
+typedef enum {
+  HS_CELL_INTRO_ACK_SUCCESS = 0x0000, /* Cell relayed to service. */
+  HS_CELL_INTRO_ACK_FAILURE = 0x0001, /* Service ID not recognized */
+  HS_CELL_INTRO_ACK_BADFMT  = 0x0002, /* Bad message format */
+  HS_CELL_INTRO_ACK_NORELAY = 0x0003, /* Can't relay cell to service */
+} hs_cell_introd_ack_status_t;
+
+/* Onion key type found in the INTRODUCE1 cell. */
+typedef enum {
+  HS_CELL_ONION_KEY_TYPE_NTOR = 1,
+} hs_cell_onion_key_type_t;
+
+/* This data structure contains data that we need to build an INTRODUCE1 cell
+ * used by the INTRODUCE1 build function. */
+typedef struct hs_cell_introduce1_data_t {
+  /* Is this a legacy introduction point? */
+  unsigned int is_legacy : 1;
+  /* (Legacy only) The encryption key for a legacy intro point. Only set if
+   * is_legacy is true. */
+  const crypto_pk_t *legacy_key;
+  /* Introduction point authentication public key. */
+  const ed25519_public_key_t *auth_pk;
+  /* Introduction point encryption public key. */
+  const curve25519_public_key_t *enc_pk;
+  /* Subcredentials of the service. */
+  const uint8_t *subcredential;
+  /* Onion public key for the ntor handshake. */
+  const curve25519_public_key_t *onion_pk;
+  /* Rendezvous cookie. */
+  const uint8_t *rendezvous_cookie;
+  /* Public key put before the encrypted data (CLIENT_PK). */
+  const curve25519_keypair_t *client_kp;
+  /* Rendezvous point link specifiers. */
+  smartlist_t *link_specifiers;
+} hs_cell_introduce1_data_t;
+
+/* This data structure contains data that we need to parse an INTRODUCE2 cell
+ * which is used by the INTRODUCE2 cell parsing function. On a successful
+ * parsing, the onion_pk and rendezvous_cookie will be populated with the
+ * computed key material from the cell data. This structure is only used during
+ * INTRO2 parsing and discarded after that. */
+typedef struct hs_cell_introduce2_data_t {
+  /*** Immutable Section: Set on structure init. ***/
+
+  /* Introduction point authentication public key. Pointer owned by the
+     introduction point object through which we received the INTRO2 cell. */
+  const ed25519_public_key_t *auth_pk;
+  /* Introduction point encryption keypair for the ntor handshake. Pointer
+     owned by the introduction point object through which we received the
+     INTRO2 cell*/
+  const curve25519_keypair_t *enc_kp;
+  /* Subcredentials of the service. Pointer owned by the descriptor that owns
+     the introduction point through which we received the INTRO2 cell. */
+  const uint8_t *subcredential;
+  /* Payload of the received encoded cell. */
+  const uint8_t *payload;
+  /* Size of the payload of the received encoded cell. */
+  size_t payload_len;
+
+  /*** Mutable Section: Set upon parsing INTRODUCE2 cell. ***/
+
+  /* Onion public key computed using the INTRODUCE2 encrypted section. */
+  curve25519_public_key_t onion_pk;
+  /* Rendezvous cookie taken from the INTRODUCE2 encrypted section. */
+  uint8_t rendezvous_cookie[REND_COOKIE_LEN];
+  /* Client public key from the INTRODUCE2 encrypted section. */
+  curve25519_public_key_t client_pk;
+  /* Link specifiers of the rendezvous point. Contains link_specifier_t. */
+  smartlist_t *link_specifiers;
+  /* Replay cache of the introduction point. */
+  replaycache_t *replay_cache;
+} hs_cell_introduce2_data_t;
+
+/* Build cell API. */
+ssize_t hs_cell_build_establish_intro(const char *circ_nonce,
+                                      const hs_service_intro_point_t *ip,
+                                      uint8_t *cell_out);
+ssize_t hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
+                                  size_t rendezvous_cookie_len,
+                                  const uint8_t *rendezvous_handshake_info,
+                                  size_t rendezvous_handshake_info_len,
+                                  uint8_t *cell_out);
+ssize_t hs_cell_build_introduce1(const hs_cell_introduce1_data_t *data,
+                                 uint8_t *cell_out);
+ssize_t hs_cell_build_establish_rendezvous(const uint8_t *rendezvous_cookie,
+                                           uint8_t *cell_out);
+
+/* Parse cell API. */
+ssize_t hs_cell_parse_intro_established(const uint8_t *payload,
+                                        size_t payload_len);
+ssize_t hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
+                                 const origin_circuit_t *circ,
+                                 const hs_service_t *service);
+int hs_cell_parse_introduce_ack(const uint8_t *payload, size_t payload_len);
+int hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
+                              uint8_t *handshake_info,
+                              size_t handshake_info_len);
+
+/* Util API. */
+void hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data);
+
+#endif /* !defined(TOR_HS_CELL_H) */
+
diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c
new file mode 100644
index 0000000000..5a2c7e4e42
--- /dev/null
+++ b/src/feature/hs/hs_circuit.c
@@ -0,0 +1,1249 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuit.c
+ **/
+
+#define HS_CIRCUIT_PRIVATE
+
+#include "or/or.h"
+#include "or/circpathbias.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/relay.h"
+#include "or/rendservice.h"
+#include "or/rephist.h"
+#include "or/router.h"
+
+#include "or/hs_cell.h"
+#include "or/hs_circuitmap.h"
+#include "or/hs_ident.h"
+#include "or/hs_ntor.h"
+#include "or/hs_service.h"
+#include "or/hs_circuit.h"
+
+/* Trunnel. */
+#include "trunnel/ed25519_cert.h"
+#include "trunnel/hs/cell_common.h"
+#include "trunnel/hs/cell_establish_intro.h"
+
+#include "or/cpath_build_state_st.h"
+#include "or/crypt_path_st.h"
+#include "or/node_st.h"
+#include "or/origin_circuit_st.h"
+
+/* A circuit is about to become an e2e rendezvous circuit. Check
+ * <b>circ_purpose</b> and ensure that it's properly set. Return true iff
+ * circuit purpose is properly set, otherwise return false. */
+static int
+circuit_purpose_is_correct_for_rend(unsigned int circ_purpose,
+                                    int is_service_side)
+{
+  if (is_service_side) {
+    if (circ_purpose != CIRCUIT_PURPOSE_S_CONNECT_REND) {
+      log_warn(LD_BUG,
+            "HS e2e circuit setup with wrong purpose (%d)", circ_purpose);
+      return 0;
+    }
+  }
+
+  if (!is_service_side) {
+    if (circ_purpose != CIRCUIT_PURPOSE_C_REND_READY &&
+        circ_purpose != CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) {
+      log_warn(LD_BUG,
+            "Client e2e circuit setup with wrong purpose (%d)", circ_purpose);
+      return 0;
+    }
+  }
+
+  return 1;
+}
+
+/* Create and return a crypt path for the final hop of a v3 prop224 rendezvous
+ * circuit. Initialize the crypt path crypto using the output material from the
+ * ntor key exchange at <b>ntor_key_seed</b>.
+ *
+ * If <b>is_service_side</b> is set, we are the hidden service and the final
+ * hop of the rendezvous circuit is the client on the other side. */
+static crypt_path_t *
+create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len,
+                  int is_service_side)
+{
+  uint8_t keys[HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN];
+  crypt_path_t *cpath = NULL;
+
+  /* Do the key expansion */
+  if (hs_ntor_circuit_key_expansion(ntor_key_seed, seed_len,
+                                    keys, sizeof(keys)) < 0) {
+    goto err;
+  }
+
+  /* Setup the cpath */
+  cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  cpath->magic = CRYPT_PATH_MAGIC;
+
+  if (circuit_init_cpath_crypto(cpath, (char*)keys, sizeof(keys),
+                                is_service_side, 1) < 0) {
+    tor_free(cpath);
+    goto err;
+  }
+
+ err:
+  memwipe(keys, 0, sizeof(keys));
+  return cpath;
+}
+
+/* We are a v2 legacy HS client: Create and return a crypt path for the hidden
+ * service on the other side of the rendezvous circuit <b>circ</b>. Initialize
+ * the crypt path crypto using the body of the RENDEZVOUS1 cell at
+ * <b>rend_cell_body</b> (which must be at least DH1024_KEY_LEN+DIGEST_LEN
+ * bytes).
+ */
+static crypt_path_t *
+create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body)
+{
+  crypt_path_t *hop = NULL;
+  char keys[DIGEST_LEN+CPATH_KEY_MATERIAL_LEN];
+
+  /* first DH1024_KEY_LEN bytes are g^y from the service. Finish the dh
+   * handshake...*/
+  tor_assert(circ->build_state);
+  tor_assert(circ->build_state->pending_final_cpath);
+  hop = circ->build_state->pending_final_cpath;
+
+  tor_assert(hop->rend_dh_handshake_state);
+  if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, hop->rend_dh_handshake_state,
+                               (char*)rend_cell_body, DH1024_KEY_LEN,
+                               keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) {
+    log_warn(LD_GENERAL, "Couldn't complete DH handshake.");
+    goto err;
+  }
+  /* ... and set up cpath. */
+  if (circuit_init_cpath_crypto(hop,
+                                keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
+                                0, 0) < 0)
+    goto err;
+
+  /* Check whether the digest is right... */
+  if (tor_memneq(keys, rend_cell_body+DH1024_KEY_LEN, DIGEST_LEN)) {
+    log_warn(LD_PROTOCOL, "Incorrect digest of key material.");
+    goto err;
+  }
+
+  /* clean up the crypto stuff we just made */
+  crypto_dh_free(hop->rend_dh_handshake_state);
+  hop->rend_dh_handshake_state = NULL;
+
+  goto done;
+
+ err:
+  hop = NULL;
+
+ done:
+  memwipe(keys, 0, sizeof(keys));
+  return hop;
+}
+
+/* Append the final <b>hop</b> to the cpath of the rend <b>circ</b>, and mark
+ * <b>circ</b> ready for use to transfer HS relay cells. */
+static void
+finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
+                      int is_service_side)
+{
+  tor_assert(circ);
+  tor_assert(hop);
+
+  /* Notify the circuit state machine that we are splicing this circuit */
+  int new_circ_purpose = is_service_side ?
+    CIRCUIT_PURPOSE_S_REND_JOINED : CIRCUIT_PURPOSE_C_REND_JOINED;
+  circuit_change_purpose(TO_CIRCUIT(circ), new_circ_purpose);
+
+  /* All is well. Extend the circuit. */
+  hop->state = CPATH_STATE_OPEN;
+  /* Set the windows to default. */
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  /* Now that this circuit has finished connecting to its destination,
+   * make sure circuit_get_open_circ_or_launch is willing to return it
+   * so we can actually use it. */
+  circ->hs_circ_has_timed_out = 0;
+
+  /* Append the hop to the cpath of this circuit */
+  onion_append_to_cpath(&circ->cpath, hop);
+
+  /* In legacy code, 'pending_final_cpath' points to the final hop we just
+   * appended to the cpath. We set the original pointer to NULL so that we
+   * don't double free it. */
+  if (circ->build_state) {
+    circ->build_state->pending_final_cpath = NULL;
+  }
+
+  /* Finally, mark circuit as ready to be used for client streams */
+  if (!is_service_side) {
+    circuit_try_attaching_streams(circ);
+  }
+}
+
+/* For a given circuit and a service introduction point object, register the
+ * intro circuit to the circuitmap. This supports legacy intro point. */
+static void
+register_intro_circ(const hs_service_intro_point_t *ip,
+                    origin_circuit_t *circ)
+{
+  tor_assert(ip);
+  tor_assert(circ);
+
+  if (ip->base.is_only_legacy) {
+    hs_circuitmap_register_intro_circ_v2_service_side(circ,
+                                                      ip->legacy_key_digest);
+  } else {
+    hs_circuitmap_register_intro_circ_v3_service_side(circ,
+                                         &ip->auth_key_kp.pubkey);
+  }
+}
+
+/* Return the number of opened introduction circuit for the given circuit that
+ * is matching its identity key. */
+static unsigned int
+count_opened_desc_intro_point_circuits(const hs_service_t *service,
+                                       const hs_service_descriptor_t *desc)
+{
+  unsigned int count = 0;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       const hs_service_intro_point_t *, ip) {
+    const circuit_t *circ;
+    const origin_circuit_t *ocirc = hs_circ_service_get_intro_circ(ip);
+    if (ocirc == NULL) {
+      continue;
+    }
+    circ = TO_CIRCUIT(ocirc);
+    tor_assert(circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+               circ->purpose == CIRCUIT_PURPOSE_S_INTRO);
+    /* Having a circuit not for the requested service is really bad. */
+    tor_assert(ed25519_pubkey_eq(&service->keys.identity_pk,
+                                 &ocirc->hs_ident->identity_pk));
+    /* Only count opened circuit and skip circuit that will be closed. */
+    if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN) {
+      count++;
+    }
+  } DIGEST256MAP_FOREACH_END;
+  return count;
+}
+
+/* From a given service, rendezvous cookie and handshake info, create a
+ * rendezvous point circuit identifier. This can't fail. */
+STATIC hs_ident_circuit_t *
+create_rp_circuit_identifier(const hs_service_t *service,
+                             const uint8_t *rendezvous_cookie,
+                             const curve25519_public_key_t *server_pk,
+                             const hs_ntor_rend_cell_keys_t *keys)
+{
+  hs_ident_circuit_t *ident;
+  uint8_t handshake_info[CURVE25519_PUBKEY_LEN + DIGEST256_LEN];
+
+  tor_assert(service);
+  tor_assert(rendezvous_cookie);
+  tor_assert(server_pk);
+  tor_assert(keys);
+
+  ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                               HS_IDENT_CIRCUIT_RENDEZVOUS);
+  /* Copy the RENDEZVOUS_COOKIE which is the unique identifier. */
+  memcpy(ident->rendezvous_cookie, rendezvous_cookie,
+         sizeof(ident->rendezvous_cookie));
+  /* Build the HANDSHAKE_INFO which looks like this:
+   *    SERVER_PK        [32 bytes]
+   *    AUTH_INPUT_MAC   [32 bytes]
+   */
+  memcpy(handshake_info, server_pk->public_key, CURVE25519_PUBKEY_LEN);
+  memcpy(handshake_info + CURVE25519_PUBKEY_LEN, keys->rend_cell_auth_mac,
+         DIGEST256_LEN);
+  tor_assert(sizeof(ident->rendezvous_handshake_info) ==
+             sizeof(handshake_info));
+  memcpy(ident->rendezvous_handshake_info, handshake_info,
+         sizeof(ident->rendezvous_handshake_info));
+  /* Finally copy the NTOR_KEY_SEED for e2e encryption on the circuit. */
+  tor_assert(sizeof(ident->rendezvous_ntor_key_seed) ==
+             sizeof(keys->ntor_key_seed));
+  memcpy(ident->rendezvous_ntor_key_seed, keys->ntor_key_seed,
+         sizeof(ident->rendezvous_ntor_key_seed));
+  return ident;
+}
+
+/* From a given service and service intro point, create an introduction point
+ * circuit identifier. This can't fail. */
+static hs_ident_circuit_t *
+create_intro_circuit_identifier(const hs_service_t *service,
+                                const hs_service_intro_point_t *ip)
+{
+  hs_ident_circuit_t *ident;
+
+  tor_assert(service);
+  tor_assert(ip);
+
+  ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                               HS_IDENT_CIRCUIT_INTRO);
+  ed25519_pubkey_copy(&ident->intro_auth_pk, &ip->auth_key_kp.pubkey);
+
+  return ident;
+}
+
+/* For a given introduction point and an introduction circuit, send the
+ * ESTABLISH_INTRO cell. The service object is used for logging. This can fail
+ * and if so, the circuit is closed and the intro point object is flagged
+ * that the circuit is not established anymore which is important for the
+ * retry mechanism. */
+static void
+send_establish_intro(const hs_service_t *service,
+                     hs_service_intro_point_t *ip, origin_circuit_t *circ)
+{
+  ssize_t cell_len;
+  uint8_t payload[RELAY_PAYLOAD_SIZE];
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(circ);
+
+  /* Encode establish intro cell. */
+  cell_len = hs_cell_build_establish_intro(circ->cpath->prev->rend_circ_nonce,
+                                           ip, payload);
+  if (cell_len < 0) {
+    log_warn(LD_REND, "Unable to encode ESTABLISH_INTRO cell for service %s "
+                      "on circuit %u. Closing circuit.",
+             safe_str_client(service->onion_address),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+
+  /* Send the cell on the circuit. */
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_ESTABLISH_INTRO,
+                                   (char *) payload, cell_len,
+                                   circ->cpath->prev) < 0) {
+    log_info(LD_REND, "Unable to send ESTABLISH_INTRO cell for service %s "
+                      "on circuit %u.",
+             safe_str_client(service->onion_address),
+             TO_CIRCUIT(circ)->n_circ_id);
+    /* On error, the circuit has been closed. */
+    goto done;
+  }
+
+  /* Record the attempt to use this circuit. */
+  pathbias_count_use_attempt(circ);
+  goto done;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+ done:
+  memwipe(payload, 0, sizeof(payload));
+}
+
+/* Return a string constant describing the anonymity of service. */
+static const char *
+get_service_anonymity_string(const hs_service_t *service)
+{
+  if (service->config.is_single_onion) {
+    return "single onion";
+  } else {
+    return "hidden";
+  }
+}
+
+/* For a given service, the ntor onion key and a rendezvous cookie, launch a
+ * circuit to the rendezvous point specified by the link specifiers. On
+ * success, a circuit identifier is attached to the circuit with the needed
+ * data. This function will try to open a circuit for a maximum value of
+ * MAX_REND_FAILURES then it will give up. */
+static void
+launch_rendezvous_point_circuit(const hs_service_t *service,
+                                const hs_service_intro_point_t *ip,
+                                const hs_cell_introduce2_data_t *data)
+{
+  int circ_needs_uptime;
+  time_t now = time(NULL);
+  extend_info_t *info = NULL;
+  origin_circuit_t *circ;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(data);
+
+  circ_needs_uptime = hs_service_requires_uptime_circ(service->config.ports);
+
+  /* Get the extend info data structure for the chosen rendezvous point
+   * specified by the given link specifiers. */
+  info = hs_get_extend_info_from_lspecs(data->link_specifiers,
+                                        &data->onion_pk,
+                                        service->config.is_single_onion);
+  if (info == NULL) {
+    /* We are done here, we can't extend to the rendezvous point.
+     * If you're running an IPv6-only v3 single onion service on 0.3.2 or with
+     * 0.3.2 clients, and somehow disable the option check, it will fail here.
+     */
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Not enough info to open a circuit to a rendezvous point for "
+           "%s service %s.",
+           get_service_anonymity_string(service),
+           safe_str_client(service->onion_address));
+    goto end;
+  }
+
+  for (int i = 0; i < MAX_REND_FAILURES; i++) {
+    int circ_flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
+    if (circ_needs_uptime) {
+      circ_flags |= CIRCLAUNCH_NEED_UPTIME;
+    }
+    /* Firewall and policies are checked when getting the extend info. */
+    if (service->config.is_single_onion) {
+      circ_flags |= CIRCLAUNCH_ONEHOP_TUNNEL;
+    }
+
+    circ = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND, info,
+                                         circ_flags);
+    if (circ != NULL) {
+      /* Stop retrying, we have a circuit! */
+      break;
+    }
+  }
+  if (circ == NULL) {
+    log_warn(LD_REND, "Giving up on launching a rendezvous circuit to %s "
+                      "for %s service %s",
+             safe_str_client(extend_info_describe(info)),
+             get_service_anonymity_string(service),
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+  log_info(LD_REND, "Rendezvous circuit launched to %s with cookie %s "
+                    "for %s service %s",
+           safe_str_client(extend_info_describe(info)),
+           safe_str_client(hex_str((const char *) data->rendezvous_cookie,
+                                   REND_COOKIE_LEN)),
+           get_service_anonymity_string(service),
+           safe_str_client(service->onion_address));
+  tor_assert(circ->build_state);
+  /* Rendezvous circuit have a specific timeout for the time spent on trying
+   * to connect to the rendezvous point. */
+  circ->build_state->expiry_time = now + MAX_REND_TIMEOUT;
+
+  /* Create circuit identifier and key material. */
+  {
+    hs_ntor_rend_cell_keys_t keys;
+    curve25519_keypair_t ephemeral_kp;
+    /* No need for extra strong, this is only for this circuit life time. This
+     * key will be used for the RENDEZVOUS1 cell that will be sent on the
+     * circuit once opened. */
+    curve25519_keypair_generate(&ephemeral_kp, 0);
+    if (hs_ntor_service_get_rendezvous1_keys(&ip->auth_key_kp.pubkey,
+                                             &ip->enc_key_kp,
+                                             &ephemeral_kp, &data->client_pk,
+                                             &keys) < 0) {
+      /* This should not really happened but just in case, don't make tor
+       * freak out, close the circuit and move on. */
+      log_info(LD_REND, "Unable to get RENDEZVOUS1 key material for "
+                        "service %s",
+               safe_str_client(service->onion_address));
+      circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+      goto end;
+    }
+    circ->hs_ident = create_rp_circuit_identifier(service,
+                                                  data->rendezvous_cookie,
+                                                  &ephemeral_kp.pubkey, &keys);
+    memwipe(&ephemeral_kp, 0, sizeof(ephemeral_kp));
+    memwipe(&keys, 0, sizeof(keys));
+    tor_assert(circ->hs_ident);
+  }
+
+ end:
+  extend_info_free(info);
+}
+
+/* Return true iff the given service rendezvous circuit circ is allowed for a
+ * relaunch to the rendezvous point. */
+static int
+can_relaunch_service_rendezvous_point(const origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  /* This is initialized when allocating an origin circuit. */
+  tor_assert(circ->build_state);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* XXX: Retrying under certain condition. This is related to #22455. */
+
+  /* Avoid to relaunch twice a circuit to the same rendezvous point at the
+   * same time. */
+  if (circ->hs_service_side_rend_circ_has_been_relaunched) {
+    log_info(LD_REND, "Rendezvous circuit to %s has already been retried. "
+                      "Skipping retry.",
+             safe_str_client(
+                  extend_info_describe(circ->build_state->chosen_exit)));
+    goto disallow;
+  }
+
+  /* We check failure_count >= hs_get_service_max_rend_failures()-1 below, and
+   * the -1 is because we increment the failure count for our current failure
+   * *after* this clause. */
+  int max_rend_failures = hs_get_service_max_rend_failures() - 1;
+
+  /* A failure count that has reached maximum allowed or circuit that expired,
+   * we skip relaunching. */
+  if (circ->build_state->failure_count > max_rend_failures ||
+      circ->build_state->expiry_time <= time(NULL)) {
+    log_info(LD_REND, "Attempt to build a rendezvous circuit to %s has "
+                      "failed with %d attempts and expiry time %ld. "
+                      "Giving up building.",
+             safe_str_client(
+                  extend_info_describe(circ->build_state->chosen_exit)),
+             circ->build_state->failure_count,
+             (long int) circ->build_state->expiry_time);
+    goto disallow;
+  }
+
+  /* Allowed to relaunch. */
+  return 1;
+ disallow:
+  return 0;
+}
+
+/* Retry the rendezvous point of circ by launching a new circuit to it. */
+static void
+retry_service_rendezvous_point(const origin_circuit_t *circ)
+{
+  int flags = 0;
+  origin_circuit_t *new_circ;
+  cpath_build_state_t *bstate;
+
+  tor_assert(circ);
+  /* This is initialized when allocating an origin circuit. */
+  tor_assert(circ->build_state);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* Ease our life. */
+  bstate = circ->build_state;
+
+  log_info(LD_REND, "Retrying rendezvous point circuit to %s",
+           safe_str_client(extend_info_describe(bstate->chosen_exit)));
+
+  /* Get the current build state flags for the next circuit. */
+  flags |= (bstate->need_uptime) ? CIRCLAUNCH_NEED_UPTIME : 0;
+  flags |= (bstate->need_capacity) ? CIRCLAUNCH_NEED_CAPACITY : 0;
+  flags |= (bstate->is_internal) ? CIRCLAUNCH_IS_INTERNAL : 0;
+
+  /* We do NOT add the onehop tunnel flag even though it might be a single
+   * onion service. The reason is that if we failed once to connect to the RP
+   * with a direct connection, we consider that chances are that we will fail
+   * again so try a 3-hop circuit and hope for the best. Because the service
+   * has no anonymity (single onion), this change of behavior won't affect
+   * security directly. */
+
+  new_circ = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND,
+                                           bstate->chosen_exit, flags);
+  if (new_circ == NULL) {
+    log_warn(LD_REND, "Failed to launch rendezvous circuit to %s",
+             safe_str_client(extend_info_describe(bstate->chosen_exit)));
+    goto done;
+  }
+
+  /* Transfer build state information to the new circuit state in part to
+   * catch any other failures. */
+  new_circ->build_state->failure_count = bstate->failure_count+1;
+  new_circ->build_state->expiry_time = bstate->expiry_time;
+  new_circ->hs_ident = hs_ident_circuit_dup(circ->hs_ident);
+
+ done:
+  return;
+}
+
+/* Add all possible link specifiers in node to lspecs.
+ * legacy ID is mandatory thus MUST be present in node. If the primary address
+ * is not IPv4, log a BUG() warning, and return an empty smartlist.
+ * Includes ed25519 id and IPv6 link specifiers if present in the node. */
+static void
+get_lspecs_from_node(const node_t *node, smartlist_t *lspecs)
+{
+  link_specifier_t *ls;
+  tor_addr_port_t ap;
+
+  tor_assert(node);
+  tor_assert(lspecs);
+
+  /* Get the relay's IPv4 address. */
+  node_get_prim_orport(node, &ap);
+
+  /* We expect the node's primary address to be a valid IPv4 address.
+   * This conforms to the protocol, which requires either an IPv4 or IPv6
+   * address (or both). */
+  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
+      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
+    return;
+  }
+
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
+  link_specifier_set_un_ipv4_port(ls, ap.port);
+  /* Four bytes IPv4 and two bytes port. */
+  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
+                            sizeof(ap.port));
+  smartlist_add(lspecs, ls);
+
+  /* Legacy ID is mandatory and will always be present in node. */
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
+         link_specifier_getlen_un_legacy_id(ls));
+  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
+  smartlist_add(lspecs, ls);
+
+  /* ed25519 ID is only included if the node has it. */
+  if (!ed25519_public_key_is_zero(&node->ed25519_id)) {
+    ls = link_specifier_new();
+    link_specifier_set_ls_type(ls, LS_ED25519_ID);
+    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
+           link_specifier_getlen_un_ed25519_id(ls));
+    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
+    smartlist_add(lspecs, ls);
+  }
+
+  /* Check for IPv6. If so, include it as well. */
+  if (node_has_ipv6_orport(node)) {
+    ls = link_specifier_new();
+    node_get_pref_ipv6_orport(node, &ap);
+    link_specifier_set_ls_type(ls, LS_IPV6);
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
+    smartlist_add(lspecs, ls);
+  }
+}
+
+/* Using the given descriptor intro point ip, the node of the
+ * rendezvous point rp_node and the service's subcredential, populate the
+ * already allocated intro1_data object with the needed key material and link
+ * specifiers.
+ *
+ * If rp_node has an invalid primary address, intro1_data->link_specifiers
+ * will be an empty list. Otherwise, this function can't fail. The ip
+ * MUST be a valid object containing the needed keys and authentication
+ * method. */
+static void
+setup_introduce1_data(const hs_desc_intro_point_t *ip,
+                      const node_t *rp_node,
+                      const uint8_t *subcredential,
+                      hs_cell_introduce1_data_t *intro1_data)
+{
+  smartlist_t *rp_lspecs;
+
+  tor_assert(ip);
+  tor_assert(rp_node);
+  tor_assert(subcredential);
+  tor_assert(intro1_data);
+
+  /* Build the link specifiers from the extend information of the rendezvous
+   * circuit that we've picked previously. */
+  rp_lspecs = smartlist_new();
+  get_lspecs_from_node(rp_node, rp_lspecs);
+
+  /* Populate the introduce1 data object. */
+  memset(intro1_data, 0, sizeof(hs_cell_introduce1_data_t));
+  if (ip->legacy.key != NULL) {
+    intro1_data->is_legacy = 1;
+    intro1_data->legacy_key = ip->legacy.key;
+  }
+  intro1_data->auth_pk = &ip->auth_key_cert->signed_key;
+  intro1_data->enc_pk = &ip->enc_key;
+  intro1_data->subcredential = subcredential;
+  intro1_data->onion_pk = node_get_curve25519_onion_key(rp_node);
+  intro1_data->link_specifiers = rp_lspecs;
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/* Return an introduction point circuit matching the given intro point object.
+ * NULL is returned is no such circuit can be found. */
+origin_circuit_t *
+hs_circ_service_get_intro_circ(const hs_service_intro_point_t *ip)
+{
+  tor_assert(ip);
+
+  if (ip->base.is_only_legacy) {
+    return hs_circuitmap_get_intro_circ_v2_service_side(ip->legacy_key_digest);
+  } else {
+    return hs_circuitmap_get_intro_circ_v3_service_side(
+                                        &ip->auth_key_kp.pubkey);
+  }
+}
+
+/* Called when we fail building a rendezvous circuit at some point other than
+ * the last hop: launches a new circuit to the same rendezvous point. This
+ * supports legacy service.
+ *
+ * We currently relaunch connections to rendezvous points if:
+ * - A rendezvous circuit timed out before connecting to RP.
+ * - The rendezvous circuit failed to connect to the RP.
+ *
+ * We avoid relaunching a connection to this rendezvous point if:
+ * - We have already tried MAX_REND_FAILURES times to connect to this RP,
+ * - We've been trying to connect to this RP for more than MAX_REND_TIMEOUT
+ *   seconds, or
+ * - We've already retried this specific rendezvous circuit.
+ */
+void
+hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* Check if we are allowed to relaunch to the rendezvous point of circ. */
+  if (!can_relaunch_service_rendezvous_point(circ)) {
+    goto done;
+  }
+
+  /* Flag the circuit that we are relaunching, to avoid to relaunch twice a
+   * circuit to the same rendezvous point at the same time. */
+  circ->hs_service_side_rend_circ_has_been_relaunched = 1;
+
+  /* Legacy services don't have a hidden service ident. */
+  if (circ->hs_ident) {
+    retry_service_rendezvous_point(circ);
+  } else {
+    rend_service_relaunch_rendezvous(circ);
+  }
+
+ done:
+  return;
+}
+
+/* For a given service and a service intro point, launch a circuit to the
+ * extend info ei. If the service is a single onion, a one-hop circuit will be
+ * requested. Return 0 if the circuit was successfully launched and tagged
+ * with the correct identifier. On error, a negative value is returned. */
+int
+hs_circ_launch_intro_point(hs_service_t *service,
+                           const hs_service_intro_point_t *ip,
+                           extend_info_t *ei)
+{
+  /* Standard flags for introduction circuit. */
+  int ret = -1, circ_flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  origin_circuit_t *circ;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(ei);
+
+  /* Update circuit flags in case of a single onion service that requires a
+   * direct connection. */
+  if (service->config.is_single_onion) {
+    circ_flags |= CIRCLAUNCH_ONEHOP_TUNNEL;
+  }
+
+  log_info(LD_REND, "Launching a circuit to intro point %s for service %s.",
+           safe_str_client(extend_info_describe(ei)),
+           safe_str_client(service->onion_address));
+
+  /* Note down the launch for the retry period. Even if the circuit fails to
+   * be launched, we still want to respect the retry period to avoid stress on
+   * the circuit subsystem. */
+  service->state.num_intro_circ_launched++;
+  circ = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO,
+                                       ei, circ_flags);
+  if (circ == NULL) {
+    goto end;
+  }
+
+  /* Setup the circuit identifier and attach it to it. */
+  circ->hs_ident = create_intro_circuit_identifier(service, ip);
+  tor_assert(circ->hs_ident);
+  /* Register circuit in the global circuitmap. */
+  register_intro_circ(ip, circ);
+
+  /* Success. */
+  ret = 0;
+ end:
+  return ret;
+}
+
+/* Called when a service introduction point circuit is done building. Given
+ * the service and intro point object, this function will send the
+ * ESTABLISH_INTRO cell on the circuit. Return 0 on success. Return 1 if the
+ * circuit has been repurposed to General because we already have too many
+ * opened. */
+int
+hs_circ_service_intro_has_opened(hs_service_t *service,
+                                 hs_service_intro_point_t *ip,
+                                 const hs_service_descriptor_t *desc,
+                                 origin_circuit_t *circ)
+{
+  int ret = 0;
+  unsigned int num_intro_circ, num_needed_circ;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(desc);
+  tor_assert(circ);
+
+  /* Cound opened circuits that have sent ESTABLISH_INTRO cells or are already
+   * established introduction circuits */
+  num_intro_circ = count_opened_desc_intro_point_circuits(service, desc);
+  num_needed_circ = service->config.num_intro_points;
+  if (num_intro_circ > num_needed_circ) {
+    /* There are too many opened valid intro circuit for what the service
+     * needs so repurpose this one. */
+
+    /* XXX: Legacy code checks options->ExcludeNodes and if not NULL it just
+     * closes the circuit. I have NO idea why it does that so it hasn't been
+     * added here. I can only assume in case our ExcludeNodes list changes but
+     * in that case, all circuit are flagged unusable (config.c). --dgoulet */
+
+    log_info(LD_CIRC | LD_REND, "Introduction circuit just opened but we "
+                                "have enough for service %s. Repurposing "
+                                "it to general and leaving internal.",
+             safe_str_client(service->onion_address));
+    tor_assert(circ->build_state->is_internal);
+    /* Remove it from the circuitmap. */
+    hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
+    /* Cleaning up the hidden service identifier and repurpose. */
+    hs_ident_circuit_free(circ->hs_ident);
+    circ->hs_ident = NULL;
+    if (circuit_should_use_vanguards(TO_CIRCUIT(circ)->purpose))
+      circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_HS_VANGUARDS);
+    else
+      circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_GENERAL);
+
+    /* Inform that this circuit just opened for this new purpose. */
+    circuit_has_opened(circ);
+    /* This return value indicate to the caller that the IP object should be
+     * removed from the service because it's corresponding circuit has just
+     * been repurposed. */
+    ret = 1;
+    goto done;
+  }
+
+  log_info(LD_REND, "Introduction circuit %u established for service %s.",
+           TO_CIRCUIT(circ)->n_circ_id,
+           safe_str_client(service->onion_address));
+  circuit_log_path(LOG_INFO, LD_REND, circ);
+
+  /* Time to send an ESTABLISH_INTRO cell on this circuit. On error, this call
+   * makes sure the circuit gets closed. */
+  send_establish_intro(service, ip, circ);
+
+ done:
+  return ret;
+}
+
+/* Called when a service rendezvous point circuit is done building. Given the
+ * service and the circuit, this function will send a RENDEZVOUS1 cell on the
+ * circuit using the information in the circuit identifier. If the cell can't
+ * be sent, the circuit is closed. */
+void
+hs_circ_service_rp_has_opened(const hs_service_t *service,
+                              origin_circuit_t *circ)
+{
+  size_t payload_len;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+
+  tor_assert(service);
+  tor_assert(circ);
+  tor_assert(circ->hs_ident);
+
+  /* Some useful logging. */
+  log_info(LD_REND, "Rendezvous circuit %u has opened with cookie %s "
+                    "for service %s",
+           TO_CIRCUIT(circ)->n_circ_id,
+           hex_str((const char *) circ->hs_ident->rendezvous_cookie,
+                   REND_COOKIE_LEN),
+           safe_str_client(service->onion_address));
+  circuit_log_path(LOG_INFO, LD_REND, circ);
+
+  /* This can't fail. */
+  payload_len = hs_cell_build_rendezvous1(
+                        circ->hs_ident->rendezvous_cookie,
+                        sizeof(circ->hs_ident->rendezvous_cookie),
+                        circ->hs_ident->rendezvous_handshake_info,
+                        sizeof(circ->hs_ident->rendezvous_handshake_info),
+                        payload);
+
+  /* Pad the payload with random bytes so it matches the size of a legacy cell
+   * which is normally always bigger. Also, the size of a legacy cell is
+   * always smaller than the RELAY_PAYLOAD_SIZE so this is safe. */
+  if (payload_len < HS_LEGACY_RENDEZVOUS_CELL_SIZE) {
+    crypto_rand((char *) payload + payload_len,
+                HS_LEGACY_RENDEZVOUS_CELL_SIZE - payload_len);
+    payload_len = HS_LEGACY_RENDEZVOUS_CELL_SIZE;
+  }
+
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_RENDEZVOUS1,
+                                   (const char *) payload, payload_len,
+                                   circ->cpath->prev) < 0) {
+    /* On error, circuit is closed. */
+    log_warn(LD_REND, "Unable to send RENDEZVOUS1 cell on circuit %u "
+                      "for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Setup end-to-end rendezvous circuit between the client and us. */
+  if (hs_circuit_setup_e2e_rend_circ(circ,
+                       circ->hs_ident->rendezvous_ntor_key_seed,
+                       sizeof(circ->hs_ident->rendezvous_ntor_key_seed),
+                       1) < 0) {
+    log_warn(LD_GENERAL, "Failed to setup circ");
+    goto done;
+  }
+
+ done:
+  memwipe(payload, 0, sizeof(payload));
+}
+
+/* Circ has been expecting an INTRO_ESTABLISHED cell that just arrived. Handle
+ * the INTRO_ESTABLISHED cell payload of length payload_len arriving on the
+ * given introduction circuit circ. The service is only used for logging
+ * purposes. Return 0 on success else a negative value. */
+int
+hs_circ_handle_intro_established(const hs_service_t *service,
+                                 const hs_service_intro_point_t *ip,
+                                 origin_circuit_t *circ,
+                                 const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(circ);
+  tor_assert(payload);
+
+  if (BUG(TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)) {
+    goto done;
+  }
+
+  /* Try to parse the payload into a cell making sure we do actually have a
+   * valid cell. For a legacy node, it's an empty payload so as long as we
+   * have the cell, we are good. */
+  if (!ip->base.is_only_legacy &&
+      hs_cell_parse_intro_established(payload, payload_len) < 0) {
+    log_warn(LD_REND, "Unable to parse the INTRO_ESTABLISHED cell on "
+                      "circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Switch the purpose to a fully working intro point. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_S_INTRO);
+  /* Getting a valid INTRODUCE_ESTABLISHED means we've successfully used the
+   * circuit so update our pathbias subsystem. */
+  pathbias_mark_use_success(circ);
+  /* Success. */
+  ret = 0;
+
+ done:
+  return ret;
+}
+
+/* We just received an INTRODUCE2 cell on the established introduction circuit
+ * circ.  Handle the INTRODUCE2 payload of size payload_len for the given
+ * circuit and service. This cell is associated with the intro point object ip
+ * and the subcredential. Return 0 on success else a negative value. */
+int
+hs_circ_handle_introduce2(const hs_service_t *service,
+                          const origin_circuit_t *circ,
+                          hs_service_intro_point_t *ip,
+                          const uint8_t *subcredential,
+                          const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+  time_t elapsed;
+  hs_cell_introduce2_data_t data;
+
+  tor_assert(service);
+  tor_assert(circ);
+  tor_assert(ip);
+  tor_assert(subcredential);
+  tor_assert(payload);
+
+  /* Populate the data structure with everything we need for the cell to be
+   * parsed, decrypted and key material computed correctly. */
+  data.auth_pk = &ip->auth_key_kp.pubkey;
+  data.enc_kp = &ip->enc_key_kp;
+  data.subcredential = subcredential;
+  data.payload = payload;
+  data.payload_len = payload_len;
+  data.link_specifiers = smartlist_new();
+  data.replay_cache = ip->replay_cache;
+
+  if (hs_cell_parse_introduce2(&data, circ, service) < 0) {
+    goto done;
+  }
+
+  /* Check whether we've seen this REND_COOKIE before to detect repeats. */
+  if (replaycache_add_test_and_elapsed(
+           service->state.replay_cache_rend_cookie,
+           data.rendezvous_cookie, sizeof(data.rendezvous_cookie),
+           &elapsed)) {
+    /* A Tor client will send a new INTRODUCE1 cell with the same REND_COOKIE
+     * as its previous one if its intro circ times out while in state
+     * CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT. If we received the first
+     * INTRODUCE1 cell (the intro-point relay converts it into an INTRODUCE2
+     * cell), we are already trying to connect to that rend point (and may
+     * have already succeeded); drop this cell. */
+    log_info(LD_REND, "We received an INTRODUCE2 cell with same REND_COOKIE "
+                      "field %ld seconds ago. Dropping cell.",
+             (long int) elapsed);
+    goto done;
+  }
+
+  /* At this point, we just confirmed that the full INTRODUCE2 cell is valid
+   * so increment our counter that we've seen one on this intro point. */
+  ip->introduce2_count++;
+
+  /* Launch rendezvous circuit with the onion key and rend cookie. */
+  launch_rendezvous_point_circuit(service, ip, &data);
+  /* Success. */
+  ret = 0;
+
+ done:
+  SMARTLIST_FOREACH(data.link_specifiers, link_specifier_t *, lspec,
+                    link_specifier_free(lspec));
+  smartlist_free(data.link_specifiers);
+  memwipe(&data, 0, sizeof(data));
+  return ret;
+}
+
+/* Circuit <b>circ</b> just finished the rend ntor key exchange. Use the key
+ * exchange output material at <b>ntor_key_seed</b> and setup <b>circ</b> to
+ * serve as a rendezvous end-to-end circuit between the client and the
+ * service. If <b>is_service_side</b> is set, then we are the hidden service
+ * and the other side is the client.
+ *
+ * Return 0 if the operation went well; in case of error return -1. */
+int
+hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
+                               const uint8_t *ntor_key_seed, size_t seed_len,
+                               int is_service_side)
+{
+  if (BUG(!circuit_purpose_is_correct_for_rend(TO_CIRCUIT(circ)->purpose,
+                                        is_service_side))) {
+    return -1;
+  }
+
+  crypt_path_t *hop = create_rend_cpath(ntor_key_seed, seed_len,
+                                        is_service_side);
+  if (!hop) {
+    log_warn(LD_REND, "Couldn't get v3 %s cpath!",
+             is_service_side ? "service-side" : "client-side");
+    return -1;
+  }
+
+  finalize_rend_circuit(circ, hop, is_service_side);
+
+  return 0;
+}
+
+/* We are a v2 legacy HS client and we just received a RENDEZVOUS1 cell
+ * <b>rend_cell_body</b> on <b>circ</b>. Finish up the DH key exchange and then
+ * extend the crypt path of <b>circ</b> so that the hidden service is on the
+ * other side. */
+int
+hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
+                                             const uint8_t *rend_cell_body)
+{
+
+  if (BUG(!circuit_purpose_is_correct_for_rend(
+                                      TO_CIRCUIT(circ)->purpose, 0))) {
+    return -1;
+  }
+
+  crypt_path_t *hop = create_rend_cpath_legacy(circ, rend_cell_body);
+  if (!hop) {
+    log_warn(LD_GENERAL, "Couldn't get v2 cpath.");
+    return -1;
+  }
+
+  finalize_rend_circuit(circ, hop, 0);
+
+  return 0;
+}
+
+/* Given the introduction circuit intro_circ, the rendezvous circuit
+ * rend_circ, a descriptor intro point object ip and the service's
+ * subcredential, send an INTRODUCE1 cell on intro_circ.
+ *
+ * This will also setup the circuit identifier on rend_circ containing the key
+ * material for the handshake and e2e encryption. Return 0 on success else
+ * negative value. Because relay_send_command_from_edge() closes the circuit
+ * on error, it is possible that intro_circ is closed on error. */
+int
+hs_circ_send_introduce1(origin_circuit_t *intro_circ,
+                        origin_circuit_t *rend_circ,
+                        const hs_desc_intro_point_t *ip,
+                        const uint8_t *subcredential)
+{
+  int ret = -1;
+  ssize_t payload_len;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+  hs_cell_introduce1_data_t intro1_data;
+
+  tor_assert(intro_circ);
+  tor_assert(rend_circ);
+  tor_assert(ip);
+  tor_assert(subcredential);
+
+  /* It is undefined behavior in hs_cell_introduce1_data_clear() if intro1_data
+   * has been declared on the stack but not initialized. Here, we set it to 0.
+   */
+  memset(&intro1_data, 0, sizeof(hs_cell_introduce1_data_t));
+
+  /* This takes various objects in order to populate the introduce1 data
+   * object which is used to build the content of the cell. */
+  const node_t *exit_node = build_state_get_exit_node(rend_circ->build_state);
+  if (exit_node == NULL) {
+    log_info(LD_REND, "Unable to get rendezvous point for circuit %u. "
+             "Failing.", TO_CIRCUIT(intro_circ)->n_circ_id);
+    goto done;
+  }
+  setup_introduce1_data(ip, exit_node, subcredential, &intro1_data);
+  /* If we didn't get any link specifiers, it's because our node was
+   * bad. */
+  if (BUG(!intro1_data.link_specifiers) ||
+      !smartlist_len(intro1_data.link_specifiers)) {
+    log_warn(LD_REND, "Unable to get link specifiers for INTRODUCE1 cell on "
+             "circuit %u.", TO_CIRCUIT(intro_circ)->n_circ_id);
+    goto done;
+  }
+
+  /* Final step before we encode a cell, we setup the circuit identifier which
+   * will generate both the rendezvous cookie and client keypair for this
+   * connection. Those are put in the ident. */
+  intro1_data.rendezvous_cookie = rend_circ->hs_ident->rendezvous_cookie;
+  intro1_data.client_kp = &rend_circ->hs_ident->rendezvous_client_kp;
+
+  memcpy(intro_circ->hs_ident->rendezvous_cookie,
+         rend_circ->hs_ident->rendezvous_cookie,
+         sizeof(intro_circ->hs_ident->rendezvous_cookie));
+
+  /* From the introduce1 data object, this will encode the INTRODUCE1 cell
+   * into payload which is then ready to be sent as is. */
+  payload_len = hs_cell_build_introduce1(&intro1_data, payload);
+  if (BUG(payload_len < 0)) {
+    goto done;
+  }
+
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(intro_circ),
+                                   RELAY_COMMAND_INTRODUCE1,
+                                   (const char *) payload, payload_len,
+                                   intro_circ->cpath->prev) < 0) {
+    /* On error, circuit is closed. */
+    log_warn(LD_REND, "Unable to send INTRODUCE1 cell on circuit %u.",
+             TO_CIRCUIT(intro_circ)->n_circ_id);
+    goto done;
+  }
+
+  /* Success. */
+  ret = 0;
+  goto done;
+
+ done:
+  hs_cell_introduce1_data_clear(&intro1_data);
+  memwipe(payload, 0, sizeof(payload));
+  return ret;
+}
+
+/* Send an ESTABLISH_RENDEZVOUS cell along the rendezvous circuit circ. On
+ * success, 0 is returned else -1 and the circuit is marked for close. */
+int
+hs_circ_send_establish_rendezvous(origin_circuit_t *circ)
+{
+  ssize_t cell_len = 0;
+  uint8_t cell[RELAY_PAYLOAD_SIZE] = {0};
+
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+
+  log_info(LD_REND, "Send an ESTABLISH_RENDEZVOUS cell on circuit %u",
+           TO_CIRCUIT(circ)->n_circ_id);
+
+  /* Set timestamp_dirty, because circuit_expire_building expects it,
+   * and the rend cookie also means we've used the circ. */
+  TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+
+  /* We've attempted to use this circuit. Probe it if we fail */
+  pathbias_count_use_attempt(circ);
+
+  /* Generate the RENDEZVOUS_COOKIE and place it in the identifier so we can
+   * complete the handshake when receiving the acknowledgement. */
+  crypto_rand((char *) circ->hs_ident->rendezvous_cookie, HS_REND_COOKIE_LEN);
+  /* Generate the client keypair. No need to be extra strong, not long term */
+  curve25519_keypair_generate(&circ->hs_ident->rendezvous_client_kp, 0);
+
+  cell_len =
+    hs_cell_build_establish_rendezvous(circ->hs_ident->rendezvous_cookie,
+                                       cell);
+  if (BUG(cell_len < 0)) {
+    goto err;
+  }
+
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_ESTABLISH_RENDEZVOUS,
+                                   (const char *) cell, cell_len,
+                                   circ->cpath->prev) < 0) {
+    /* Circuit has been marked for close */
+    log_warn(LD_REND, "Unable to send ESTABLISH_RENDEZVOUS cell on "
+                      "circuit %u", TO_CIRCUIT(circ)->n_circ_id);
+    memwipe(cell, 0, cell_len);
+    goto err;
+  }
+
+  memwipe(cell, 0, cell_len);
+  return 0;
+ err:
+  return -1;
+}
+
+/* We are about to close or free this <b>circ</b>. Clean it up from any
+ * related HS data structures. This function can be called multiple times
+ * safely for the same circuit. */
+void
+hs_circ_cleanup(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* If it's a service-side intro circ, notify the HS subsystem for the intro
+   * point circuit closing so it can be dealt with cleanly. */
+  if (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+      circ->purpose == CIRCUIT_PURPOSE_S_INTRO) {
+    hs_service_intro_circ_has_closed(TO_ORIGIN_CIRCUIT(circ));
+  }
+
+  /* Clear HS circuitmap token for this circ (if any). Very important to be
+   * done after the HS subsystem has been notified of the close else the
+   * circuit will not be found.
+   *
+   * We do this at the close if possible because from that point on, the
+   * circuit is good as dead. We can't rely on removing it in the circuit
+   * free() function because we open a race window between the close and free
+   * where we can't register a new circuit for the same intro point. */
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+}
diff --git a/src/feature/hs/hs_circuit.h b/src/feature/hs/hs_circuit.h
new file mode 100644
index 0000000000..425070f4ca
--- /dev/null
+++ b/src/feature/hs/hs_circuit.h
@@ -0,0 +1,75 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuit.h
+ * \brief Header file containing circuit data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_CIRCUIT_H
+#define TOR_HS_CIRCUIT_H
+
+#include "or/or.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "or/hs_service.h"
+
+/* Cleanup function when the circuit is closed or/and freed. */
+void hs_circ_cleanup(circuit_t *circ);
+
+/* Circuit API. */
+int hs_circ_service_intro_has_opened(hs_service_t *service,
+                                     hs_service_intro_point_t *ip,
+                                     const hs_service_descriptor_t *desc,
+                                     origin_circuit_t *circ);
+void hs_circ_service_rp_has_opened(const hs_service_t *service,
+                                   origin_circuit_t *circ);
+int hs_circ_launch_intro_point(hs_service_t *service,
+                               const hs_service_intro_point_t *ip,
+                               extend_info_t *ei);
+int hs_circ_launch_rendezvous_point(const hs_service_t *service,
+                                    const curve25519_public_key_t *onion_key,
+                                    const uint8_t *rendezvous_cookie);
+void hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ);
+
+origin_circuit_t *hs_circ_service_get_intro_circ(
+                                      const hs_service_intro_point_t *ip);
+
+/* Cell API. */
+int hs_circ_handle_intro_established(const hs_service_t *service,
+                                     const hs_service_intro_point_t *ip,
+                                     origin_circuit_t *circ,
+                                     const uint8_t *payload,
+                                     size_t payload_len);
+int hs_circ_handle_introduce2(const hs_service_t *service,
+                              const origin_circuit_t *circ,
+                              hs_service_intro_point_t *ip,
+                              const uint8_t *subcredential,
+                              const uint8_t *payload, size_t payload_len);
+int hs_circ_send_introduce1(origin_circuit_t *intro_circ,
+                            origin_circuit_t *rend_circ,
+                            const hs_desc_intro_point_t *ip,
+                            const uint8_t *subcredential);
+int hs_circ_send_establish_rendezvous(origin_circuit_t *circ);
+
+/* e2e circuit API. */
+
+int hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
+                                   const uint8_t *ntor_key_seed,
+                                   size_t seed_len,
+                                   int is_service_side);
+int hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
+                                          const uint8_t *rend_cell_body);
+
+#ifdef HS_CIRCUIT_PRIVATE
+
+STATIC hs_ident_circuit_t *
+create_rp_circuit_identifier(const hs_service_t *service,
+                             const uint8_t *rendezvous_cookie,
+                             const curve25519_public_key_t *server_pk,
+                             const hs_ntor_rend_cell_keys_t *keys);
+
+#endif /* defined(HS_CIRCUIT_PRIVATE) */
+
+#endif /* !defined(TOR_HS_CIRCUIT_H) */
+
diff --git a/src/feature/hs/hs_circuitmap.c b/src/feature/hs/hs_circuitmap.c
new file mode 100644
index 0000000000..c4bf9fab43
--- /dev/null
+++ b/src/feature/hs/hs_circuitmap.c
@@ -0,0 +1,585 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuitmap.c
+ *
+ * \brief Hidden service circuitmap: A hash table that maps binary tokens to
+ *  introduction and rendezvous circuits; it's used:
+ *  (a) by relays acting as intro points and rendezvous points
+ *  (b) by hidden services to find intro and rend circuits and
+ *  (c) by HS clients to find rendezvous circuits.
+ **/
+
+#define HS_CIRCUITMAP_PRIVATE
+
+#include "or/or.h"
+#include "or/config.h"
+#include "or/circuitlist.h"
+#include "or/hs_circuitmap.h"
+
+#include "or/or_circuit_st.h"
+#include "or/origin_circuit_st.h"
+
+/************************** HS circuitmap code *******************************/
+
+/* This is the hidden service circuitmap. It's a hash table that maps
+   introduction and rendezvous tokens to specific circuits such that given a
+   token it's easy to find the corresponding circuit. */
+static struct hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+/* This is a helper function used by the hash table code (HT_). It returns 1 if
+ * two circuits have the same HS token. */
+static int
+hs_circuits_have_same_token(const circuit_t *first_circuit,
+                            const circuit_t *second_circuit)
+{
+  const hs_token_t *first_token;
+  const hs_token_t *second_token;
+
+  tor_assert(first_circuit);
+  tor_assert(second_circuit);
+
+  first_token = first_circuit->hs_token;
+  second_token = second_circuit->hs_token;
+
+  /* Both circs must have a token */
+  if (BUG(!first_token) || BUG(!second_token)) {
+    return 0;
+  }
+
+  if (first_token->type != second_token->type) {
+    return 0;
+  }
+
+  if (first_token->token_len != second_token->token_len)
+    return 0;
+
+  return tor_memeq(first_token->token,
+                   second_token->token,
+                   first_token->token_len);
+}
+
+/* This is a helper function for the hash table code (HT_). It hashes a circuit
+ * HS token into an unsigned int for use as a key by the hash table routines.*/
+static inline unsigned int
+hs_circuit_hash_token(const circuit_t *circuit)
+{
+  tor_assert(circuit->hs_token);
+
+  return (unsigned) siphash24g(circuit->hs_token->token,
+                               circuit->hs_token->token_len);
+}
+
+/* Register the circuitmap hash table */
+HT_PROTOTYPE(hs_circuitmap_ht, // The name of the hashtable struct
+             circuit_t,    // The name of the element struct,
+             hs_circuitmap_node,        // The name of HT_ENTRY member
+             hs_circuit_hash_token, hs_circuits_have_same_token)
+
+HT_GENERATE2(hs_circuitmap_ht, circuit_t, hs_circuitmap_node,
+             hs_circuit_hash_token, hs_circuits_have_same_token,
+             0.6, tor_reallocarray, tor_free_)
+
+#ifdef TOR_UNIT_TESTS
+
+/* Return the global HS circuitmap. Used by unittests. */
+hs_circuitmap_ht *
+get_hs_circuitmap(void)
+{
+  return the_hs_circuitmap;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/****************** HS circuitmap utility functions **************************/
+
+/** Return a new HS token of type <b>type</b> containing <b>token</b>. */
+static hs_token_t *
+hs_token_new(hs_token_type_t type, size_t token_len,
+             const uint8_t *token)
+{
+  tor_assert(token);
+
+  hs_token_t *hs_token = tor_malloc_zero(sizeof(hs_token_t));
+  hs_token->type = type;
+  hs_token->token_len = token_len;
+  hs_token->token = tor_memdup(token, token_len);
+
+  return hs_token;
+}
+
+#define hs_token_free(val) \
+  FREE_AND_NULL(hs_token_t, hs_token_free_, (val))
+
+/** Free memory allocated by this <b>hs_token</b>. */
+static void
+hs_token_free_(hs_token_t *hs_token)
+{
+  if (!hs_token) {
+    return;
+  }
+
+  tor_free(hs_token->token);
+  tor_free(hs_token);
+}
+
+/** Return the circuit from the circuitmap with token <b>search_token</b>. */
+static circuit_t *
+get_circuit_with_token(hs_token_t *search_token)
+{
+  tor_assert(the_hs_circuitmap);
+
+  /* We use a dummy circuit object for the hash table search routine. */
+  circuit_t search_circ;
+  search_circ.hs_token = search_token;
+  return HT_FIND(hs_circuitmap_ht, the_hs_circuitmap, &search_circ);
+}
+
+/* Helper function that registers <b>circ</b> with <b>token</b> on the HS
+   circuitmap. This function steals reference of <b>token</b>. */
+static void
+hs_circuitmap_register_impl(circuit_t *circ, hs_token_t *token)
+{
+  tor_assert(circ);
+  tor_assert(token);
+  tor_assert(the_hs_circuitmap);
+
+  /* If this circuit already has a token, clear it. */
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+
+  /* Kill old circuits with the same token. We want new intro/rend circuits to
+     take precedence over old ones, so that HSes and clients and reestablish
+     killed circuits without changing the HS token. */
+  {
+    circuit_t *found_circ;
+    found_circ = get_circuit_with_token(token);
+    if (found_circ) {
+      hs_circuitmap_remove_circuit(found_circ);
+      if (!found_circ->marked_for_close) {
+        circuit_mark_for_close(found_circ, END_CIRC_REASON_FINISHED);
+      }
+    }
+  }
+
+  /* Register circuit and token to circuitmap. */
+  circ->hs_token = token;
+  HT_INSERT(hs_circuitmap_ht, the_hs_circuitmap, circ);
+}
+
+/** Helper function: Register <b>circ</b> of <b>type</b> on the HS
+ *  circuitmap. Use the HS <b>token</b> as the key to the hash table.  If
+ *  <b>token</b> is not set, clear the circuit of any HS tokens. */
+static void
+hs_circuitmap_register_circuit(circuit_t *circ,
+                               hs_token_type_t type, size_t token_len,
+                               const uint8_t *token)
+{
+  hs_token_t *hs_token = NULL;
+
+  /* Create a new token and register it to the circuitmap */
+  tor_assert(token);
+  hs_token = hs_token_new(type, token_len, token);
+  tor_assert(hs_token);
+  hs_circuitmap_register_impl(circ, hs_token);
+}
+
+/* Helper function for hs_circuitmap_get_origin_circuit() and
+ * hs_circuitmap_get_or_circuit(). Because only circuit_t are indexed in the
+ * circuitmap, this function returns object type so the specialized functions
+ * using this helper can upcast it to the right type.
+ *
+ * Return NULL if not such circuit is found. */
+static circuit_t *
+hs_circuitmap_get_circuit_impl(hs_token_type_t type,
+                               size_t token_len,
+                               const uint8_t *token,
+                               uint8_t wanted_circ_purpose)
+{
+  circuit_t *found_circ = NULL;
+
+  tor_assert(the_hs_circuitmap);
+
+  /* Check the circuitmap if we have a circuit with this token */
+  {
+    hs_token_t *search_hs_token = hs_token_new(type, token_len, token);
+    tor_assert(search_hs_token);
+    found_circ = get_circuit_with_token(search_hs_token);
+    hs_token_free(search_hs_token);
+  }
+
+  /* Check that the circuit is useful to us */
+  if (!found_circ ||
+      found_circ->purpose != wanted_circ_purpose ||
+      found_circ->marked_for_close) {
+    return NULL;
+  }
+
+  return found_circ;
+}
+
+/* Helper function: Query circuitmap for origin circuit with <b>token</b> of
+ * size <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose
+ * equal to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked
+ * for close. Return NULL if no such circuit is found. */
+static origin_circuit_t *
+hs_circuitmap_get_origin_circuit(hs_token_type_t type,
+                                 size_t token_len,
+                                 const uint8_t *token,
+                                 uint8_t wanted_circ_purpose)
+{
+  circuit_t *circ;
+  tor_assert(token);
+  tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose));
+
+  circ = hs_circuitmap_get_circuit_impl(type, token_len, token,
+                                        wanted_circ_purpose);
+  if (!circ) {
+    return NULL;
+  }
+
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+  return TO_ORIGIN_CIRCUIT(circ);
+}
+
+/* Helper function: Query circuitmap for OR circuit with <b>token</b> of size
+ * <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose equal
+ * to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked for
+ * close. Return NULL if no such circuit is found. */
+static or_circuit_t *
+hs_circuitmap_get_or_circuit(hs_token_type_t type,
+                             size_t token_len,
+                             const uint8_t *token,
+                             uint8_t wanted_circ_purpose)
+{
+  circuit_t *circ;
+  tor_assert(token);
+  tor_assert(!CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose));
+
+  circ = hs_circuitmap_get_circuit_impl(type, token_len, token,
+                                        wanted_circ_purpose);
+  if (!circ) {
+    return NULL;
+  }
+
+  tor_assert(CIRCUIT_IS_ORCIRC(circ));
+  return TO_OR_CIRCUIT(circ);
+}
+
+/************** Public circuitmap API ****************************************/
+
+/**** Public relay-side getters: */
+
+/* Public function: Return a v3 introduction circuit to this relay with
+ * <b>auth_key</b>. Return NULL if no such circuit is found in the
+ * circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_intro_circ_v3_relay_side(
+                                          const ed25519_public_key_t *auth_key)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V3_RELAY_SIDE,
+                                      ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                      CIRCUIT_PURPOSE_INTRO_POINT);
+}
+
+/* Public function: Return v2 introduction circuit to this relay with
+ * <b>digest</b>. Return NULL if no such circuit is found in the circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V2_RELAY_SIDE,
+                                      REND_TOKEN_LEN, digest,
+                                      CIRCUIT_PURPOSE_INTRO_POINT);
+}
+
+/* Public function: Return rendezvous circuit to this relay with rendezvous
+ * <b>cookie</b>. Return NULL if no such circuit is found in the circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_REND_RELAY_SIDE,
+                                      REND_TOKEN_LEN, cookie,
+                                      CIRCUIT_PURPOSE_REND_POINT_WAITING);
+}
+
+/** Public relay-side setters: */
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_rend_circ_relay_side(or_circuit_t *circ,
+                                            const uint8_t *cookie)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_REND_RELAY_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v2_relay_side(or_circuit_t *circ,
+                                                const uint8_t *digest)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V2_RELAY_SIDE,
+                                 REND_TOKEN_LEN, digest);
+}
+
+/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v3_relay_side(or_circuit_t *circ,
+                                          const ed25519_public_key_t *auth_key)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V3_RELAY_SIDE,
+                                 ED25519_PUBKEY_LEN, auth_key->pubkey);
+}
+
+/**** Public servide-side getters: */
+
+/* Public function: Return v3 introduction circuit with <b>auth_key</b>
+ * originating from this hidden service. Return NULL if no such circuit is
+ * found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_intro_circ_v3_service_side(const
+                                             ed25519_public_key_t *auth_key)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Check first for established intro circuits */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                          ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                          CIRCUIT_PURPOSE_S_INTRO);
+  if (circ) {
+    return circ;
+  }
+
+  /* ...if nothing found, check for pending intro circs */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                          ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                          CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  return circ;
+}
+
+/* Public function: Return v2 introduction circuit originating from this hidden
+ * service with <b>digest</b>. Return NULL if no such circuit is found in the
+ * circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Check first for established intro circuits */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, digest,
+                                          CIRCUIT_PURPOSE_S_INTRO);
+  if (circ) {
+    return circ;
+  }
+
+  /* ...if nothing found, check for pending intro circs */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, digest,
+                                          CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  return circ;
+}
+
+/* Public function: Return rendezvous circuit originating from this hidden
+ * service with rendezvous <b>cookie</b>. Return NULL if no such circuit is
+ * found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Try to check if we have a connecting circuit. */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_S_CONNECT_REND);
+  if (circ) {
+    return circ;
+  }
+
+  /* Then try for connected circuit. */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_S_REND_JOINED);
+  return circ;
+}
+
+/* Public function: Return client-side rendezvous circuit with rendezvous
+ * <b>cookie</b>. It will look for circuits with the following purposes:
+
+ * a) CIRCUIT_PURPOSE_C_REND_READY: Established rend circuit (received
+ *    RENDEZVOUS_ESTABLISHED). Waiting for RENDEZVOUS2 from service, and for
+ *    INTRODUCE_ACK from intro point.
+ *
+ * b) CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: Established rend circuit and
+ *    introduce circuit acked. Waiting for RENDEZVOUS2 from service.
+ *
+ * c) CIRCUIT_PURPOSE_C_REND_JOINED: Established rend circuit and received
+ *    RENDEZVOUS2 from service.
+ *
+ * d) CIRCUIT_PURPOSE_C_ESTABLISH_REND: Rend circuit open but not yet
+ *    established.
+ *
+ * Return NULL if no such circuit is found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_rend_circ_client_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = hs_circuitmap_get_established_rend_circ_client_side(cookie);
+  if (circ) {
+    return circ;
+  }
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+  return circ;
+}
+
+/*  Public function: Return client-side established rendezvous circuit with
+ *  rendezvous <b>cookie</b>. It will look for circuits with the following
+ *  purposes:
+ *
+ * a) CIRCUIT_PURPOSE_C_REND_READY: Established rend circuit (received
+ *    RENDEZVOUS_ESTABLISHED). Waiting for RENDEZVOUS2 from service, and for
+ *    INTRODUCE_ACK from intro point.
+ *
+ * b) CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: Established rend circuit and
+ *    introduce circuit acked. Waiting for RENDEZVOUS2 from service.
+ *
+ * c) CIRCUIT_PURPOSE_C_REND_JOINED: Established rend circuit and received
+ *    RENDEZVOUS2 from service.
+ *
+ * Return NULL if no such circuit is found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_established_rend_circ_client_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_C_REND_READY);
+  if (circ) {
+    return circ;
+  }
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                 CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+  if (circ) {
+    return circ;
+  }
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_C_REND_JOINED);
+  return circ;
+}
+
+/**** Public servide-side setters: */
+
+/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v2_service_side(origin_circuit_t *circ,
+                                                  const uint8_t *digest)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                 REND_TOKEN_LEN, digest);
+}
+
+/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v3_service_side(origin_circuit_t *circ,
+                                          const ed25519_public_key_t *auth_key)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                 ED25519_PUBKEY_LEN, auth_key->pubkey);
+}
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_rend_circ_service_side(origin_circuit_t *circ,
+                                              const uint8_t *cookie)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_REND_SERVICE_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * client-side circuitmap. */
+void
+hs_circuitmap_register_rend_circ_client_side(origin_circuit_t *or_circ,
+                                             const uint8_t *cookie)
+{
+  circuit_t *circ = TO_CIRCUIT(or_circ);
+  { /* Basic circ purpose sanity checking */
+    tor_assert_nonfatal(circ->purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+  }
+
+  hs_circuitmap_register_circuit(circ, HS_TOKEN_REND_CLIENT_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+
+/**** Misc public functions: */
+
+/** Public function: Remove this circuit from the HS circuitmap. Clear its HS
+ *  token, and remove it from the hashtable. */
+void
+hs_circuitmap_remove_circuit(circuit_t *circ)
+{
+  tor_assert(the_hs_circuitmap);
+
+  if (!circ || !circ->hs_token) {
+    return;
+  }
+
+  /* Remove circ from circuitmap */
+  circuit_t *tmp;
+  tmp = HT_REMOVE(hs_circuitmap_ht, the_hs_circuitmap, circ);
+  /* ... and ensure the removal was successful. */
+  if (tmp) {
+    tor_assert(tmp == circ);
+  } else {
+    log_warn(LD_BUG, "Could not find circuit (%u) in circuitmap.",
+             circ->n_circ_id);
+  }
+
+  /* Clear token from circ */
+  hs_token_free(circ->hs_token);
+  circ->hs_token = NULL;
+}
+
+/* Public function: Initialize the global HS circuitmap. */
+void
+hs_circuitmap_init(void)
+{
+  tor_assert(!the_hs_circuitmap);
+
+  the_hs_circuitmap = tor_malloc_zero(sizeof(struct hs_circuitmap_ht));
+  HT_INIT(hs_circuitmap_ht, the_hs_circuitmap);
+}
+
+/* Public function: Free all memory allocated by the global HS circuitmap. */
+void
+hs_circuitmap_free_all(void)
+{
+  if (the_hs_circuitmap) {
+    HT_CLEAR(hs_circuitmap_ht, the_hs_circuitmap);
+    tor_free(the_hs_circuitmap);
+  }
+}
diff --git a/src/feature/hs/hs_circuitmap.h b/src/feature/hs/hs_circuitmap.h
new file mode 100644
index 0000000000..c39a37c052
--- /dev/null
+++ b/src/feature/hs/hs_circuitmap.h
@@ -0,0 +1,112 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuitmap.h
+ * \brief Header file for hs_circuitmap.c.
+ **/
+
+#ifndef TOR_HS_CIRCUITMAP_H
+#define TOR_HS_CIRCUITMAP_H
+
+typedef HT_HEAD(hs_circuitmap_ht, circuit_t) hs_circuitmap_ht;
+
+typedef struct hs_token_t hs_token_t;
+struct or_circuit_t;
+struct origin_circuit_t;
+
+/** Public HS circuitmap API: */
+
+/** Public relay-side API: */
+
+struct or_circuit_t *
+hs_circuitmap_get_intro_circ_v3_relay_side(const
+                                           ed25519_public_key_t *auth_key);
+struct or_circuit_t *
+hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest);
+struct or_circuit_t *
+hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie);
+
+void hs_circuitmap_register_rend_circ_relay_side(struct or_circuit_t *circ,
+                                                 const uint8_t *cookie);
+void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ,
+                                                     const uint8_t *digest);
+void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ,
+                                         const ed25519_public_key_t *auth_key);
+
+/** Public service-side API: */
+
+struct origin_circuit_t *
+hs_circuitmap_get_intro_circ_v3_service_side(const
+                                             ed25519_public_key_t *auth_key);
+struct origin_circuit_t *
+hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest);
+struct origin_circuit_t *
+hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie);
+struct origin_circuit_t *
+hs_circuitmap_get_rend_circ_client_side(const uint8_t *cookie);
+struct origin_circuit_t *
+hs_circuitmap_get_established_rend_circ_client_side(const uint8_t *cookie);
+
+void hs_circuitmap_register_intro_circ_v2_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const uint8_t *digest);
+void hs_circuitmap_register_intro_circ_v3_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const ed25519_public_key_t *auth_key);
+void hs_circuitmap_register_rend_circ_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const uint8_t *cookie);
+void hs_circuitmap_register_rend_circ_client_side(
+                                      struct origin_circuit_t *circ,
+                                      const uint8_t *cookie);
+
+void hs_circuitmap_remove_circuit(struct circuit_t *circ);
+
+void hs_circuitmap_init(void);
+void hs_circuitmap_free_all(void);
+
+#ifdef HS_CIRCUITMAP_PRIVATE
+
+/** Represents the type of HS token. */
+typedef enum {
+  /** A rendezvous cookie on a relay (128bit)*/
+  HS_TOKEN_REND_RELAY_SIDE,
+  /** A v2 introduction point pubkey on a relay (160bit) */
+  HS_TOKEN_INTRO_V2_RELAY_SIDE,
+  /** A v3 introduction point pubkey on a relay (256bit) */
+  HS_TOKEN_INTRO_V3_RELAY_SIDE,
+
+  /** A rendezvous cookie on a hidden service (128bit)*/
+  HS_TOKEN_REND_SERVICE_SIDE,
+  /** A v2 introduction point pubkey on a hidden service (160bit) */
+  HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+  /** A v3 introduction point pubkey on a hidden service (256bit) */
+  HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+
+  /** A rendezvous cookie on the client side (128bit) */
+  HS_TOKEN_REND_CLIENT_SIDE,
+} hs_token_type_t;
+
+/** Represents a token used in the HS protocol. Each such token maps to a
+ *  specific introduction or rendezvous circuit. */
+struct hs_token_t {
+  /* Type of HS token. */
+  hs_token_type_t type;
+
+  /* The size of the token (depends on the type). */
+  size_t token_len;
+
+  /* The token itself. Memory allocated at runtime. */
+  uint8_t *token;
+};
+
+#endif /* defined(HS_CIRCUITMAP_PRIVATE) */
+
+#ifdef TOR_UNIT_TESTS
+
+hs_circuitmap_ht *get_hs_circuitmap(void);
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* !defined(TOR_HS_CIRCUITMAP_H) */
diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c
new file mode 100644
index 0000000000..cc461e368d
--- /dev/null
+++ b/src/feature/hs/hs_client.c
@@ -0,0 +1,1623 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_client.c
+ * \brief Implement next generation hidden service client functionality
+ **/
+
+#define HS_CLIENT_PRIVATE
+
+#include "or/or.h"
+#include "or/circpathbias.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/connection_edge.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/directory.h"
+#include "or/hs_cache.h"
+#include "or/hs_cell.h"
+#include "or/hs_circuit.h"
+#include "or/hs_circuitmap.h"
+#include "or/hs_client.h"
+#include "or/hs_control.h"
+#include "or/hs_descriptor.h"
+#include "or/hs_ident.h"
+#include "or/hs_ntor.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/reasons.h"
+#include "or/rendclient.h"
+#include "or/router.h"
+#include "or/routerset.h"
+
+#include "or/cpath_build_state_st.h"
+#include "or/dir_connection_st.h"
+#include "or/entry_connection_st.h"
+#include "or/extend_info_st.h"
+#include "or/origin_circuit_st.h"
+
+/* Return a human-readable string for the client fetch status code. */
+static const char *
+fetch_status_to_string(hs_client_fetch_status_t status)
+{
+  switch (status) {
+  case HS_CLIENT_FETCH_ERROR:
+    return "Internal error";
+  case HS_CLIENT_FETCH_LAUNCHED:
+    return "Descriptor fetch launched";
+  case HS_CLIENT_FETCH_HAVE_DESC:
+    return "Already have descriptor";
+  case HS_CLIENT_FETCH_NO_HSDIRS:
+    return "No more HSDir available to query";
+  case HS_CLIENT_FETCH_NOT_ALLOWED:
+    return "Fetching descriptors is not allowed";
+  case HS_CLIENT_FETCH_MISSING_INFO:
+    return "Missing directory information";
+  case HS_CLIENT_FETCH_PENDING:
+    return "Pending descriptor fetch";
+  default:
+    return "(Unknown client fetch status code)";
+  }
+}
+
+/* Return true iff tor should close the SOCKS request(s) for the descriptor
+ * fetch that ended up with this given status code. */
+static int
+fetch_status_should_close_socks(hs_client_fetch_status_t status)
+{
+  switch (status) {
+  case HS_CLIENT_FETCH_NO_HSDIRS:
+    /* No more HSDir to query, we can't complete the SOCKS request(s). */
+  case HS_CLIENT_FETCH_ERROR:
+    /* The fetch triggered an internal error. */
+  case HS_CLIENT_FETCH_NOT_ALLOWED:
+    /* Client is not allowed to fetch (FetchHidServDescriptors 0). */
+    goto close;
+  case HS_CLIENT_FETCH_MISSING_INFO:
+  case HS_CLIENT_FETCH_HAVE_DESC:
+  case HS_CLIENT_FETCH_PENDING:
+  case HS_CLIENT_FETCH_LAUNCHED:
+    /* The rest doesn't require tor to close the SOCKS request(s). */
+    goto no_close;
+  }
+
+ no_close:
+  return 0;
+ close:
+  return 1;
+}
+
+/* Cancel all descriptor fetches currently in progress. */
+static void
+cancel_descriptor_fetches(void)
+{
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    const hs_ident_dir_conn_t *ident = TO_DIR_CONN(conn)->hs_ident;
+    if (BUG(ident == NULL)) {
+      /* A directory connection fetching a service descriptor can't have an
+       * empty hidden service identifier. */
+      continue;
+    }
+    log_debug(LD_REND, "Marking for close a directory connection fetching "
+                       "a hidden service descriptor for service %s.",
+              safe_str_client(ed25519_fmt(&ident->identity_pk)));
+    connection_mark_for_close(conn);
+  } SMARTLIST_FOREACH_END(conn);
+
+  /* No ownership of the objects in this list. */
+  smartlist_free(conns);
+  log_info(LD_REND, "Hidden service client descriptor fetches cancelled.");
+}
+
+/* Get all connections that are waiting on a circuit and flag them back to
+ * waiting for a hidden service descriptor for the given service key
+ * service_identity_pk. */
+static void
+flag_all_conn_wait_desc(const ed25519_public_key_t *service_identity_pk)
+{
+  tor_assert(service_identity_pk);
+
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    edge_connection_t *edge_conn;
+    if (BUG(!CONN_IS_EDGE(conn))) {
+      continue;
+    }
+    edge_conn = TO_EDGE_CONN(conn);
+    if (edge_conn->hs_ident &&
+        ed25519_pubkey_eq(&edge_conn->hs_ident->identity_pk,
+                          service_identity_pk)) {
+      connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
+      conn->state = AP_CONN_STATE_RENDDESC_WAIT;
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  smartlist_free(conns);
+}
+
+/* Remove tracked HSDir requests from our history for this hidden service
+ * identity public key. */
+static void
+purge_hid_serv_request(const ed25519_public_key_t *identity_pk)
+{
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+  ed25519_public_key_t blinded_pk;
+
+  tor_assert(identity_pk);
+
+  /* Get blinded pubkey of hidden service. It is possible that we just moved
+   * to a new time period meaning that we won't be able to purge the request
+   * from the previous time period. That is fine because they will expire at
+   * some point and we don't care about those anymore. */
+  hs_build_blinded_pubkey(identity_pk, NULL, 0,
+                          hs_get_time_period_num(0), &blinded_pk);
+  if (BUG(ed25519_public_to_base64(base64_blinded_pk, &blinded_pk) < 0)) {
+    return;
+  }
+  /* Purge last hidden service request from cache for this blinded key. */
+  hs_purge_hid_serv_from_last_hid_serv_requests(base64_blinded_pk);
+}
+
+/* Return true iff there is at least one pending directory descriptor request
+ * for the service identity_pk. */
+static int
+directory_request_is_pending(const ed25519_public_key_t *identity_pk)
+{
+  int ret = 0;
+  smartlist_t *conns =
+    connection_list_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    const hs_ident_dir_conn_t *ident = TO_DIR_CONN(conn)->hs_ident;
+    if (BUG(ident == NULL)) {
+      /* A directory connection fetching a service descriptor can't have an
+       * empty hidden service identifier. */
+      continue;
+    }
+    if (!ed25519_pubkey_eq(identity_pk, &ident->identity_pk)) {
+      continue;
+    }
+    ret = 1;
+    break;
+  } SMARTLIST_FOREACH_END(conn);
+
+  /* No ownership of the objects in this list. */
+  smartlist_free(conns);
+  return ret;
+}
+
+/* We failed to fetch a descriptor for the service with <b>identity_pk</b>
+ * because of <b>status</b>. Find all pending SOCKS connections for this
+ * service that are waiting on the descriptor and close them with
+ * <b>reason</b>. */
+static void
+close_all_socks_conns_waiting_for_desc(const ed25519_public_key_t *identity_pk,
+                                       hs_client_fetch_status_t status,
+                                       int reason)
+{
+  unsigned int count = 0;
+  time_t now = approx_time();
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
+    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+
+    /* Only consider the entry connections that matches the service for which
+     * we tried to get the descriptor */
+    if (!edge_conn->hs_ident ||
+        !ed25519_pubkey_eq(identity_pk,
+                           &edge_conn->hs_ident->identity_pk)) {
+      continue;
+    }
+    assert_connection_ok(base_conn, now);
+    /* Unattach the entry connection which will close for the reason. */
+    connection_mark_unattached_ap(entry_conn, reason);
+    count++;
+  } SMARTLIST_FOREACH_END(base_conn);
+
+  if (count > 0) {
+    char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    hs_build_address(identity_pk, HS_VERSION_THREE, onion_address);
+    log_notice(LD_REND, "Closed %u streams for service %s.onion "
+                        "for reason %s. Fetch status: %s.",
+               count, safe_str_client(onion_address),
+               stream_end_reason_to_string(reason),
+               fetch_status_to_string(status));
+  }
+
+  /* No ownership of the object(s) in this list. */
+  smartlist_free(conns);
+}
+
+/* Find all pending SOCKS connection waiting for a descriptor and retry them
+ * all. This is called when the directory information changed. */
+static void
+retry_all_socks_conn_waiting_for_desc(void)
+{
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    hs_client_fetch_status_t status;
+    const edge_connection_t *edge_conn =
+      ENTRY_TO_EDGE_CONN(TO_ENTRY_CONN(base_conn));
+
+    /* Ignore non HS or non v3 connection. */
+    if (edge_conn->hs_ident == NULL) {
+      continue;
+    }
+    /* In this loop, we will possibly try to fetch a descriptor for the
+     * pending connections because we just got more directory information.
+     * However, the refetch process can cleanup all SOCKS request so the same
+     * service if an internal error happens. Thus, we can end up with closed
+     * connections in our list. */
+    if (base_conn->marked_for_close) {
+      continue;
+    }
+
+    /* XXX: There is an optimization we could do which is that for a service
+     * key, we could check if we can fetch and remember that decision. */
+
+    /* Order a refetch in case it works this time. */
+    status = hs_client_refetch_hsdesc(&edge_conn->hs_ident->identity_pk);
+    if (BUG(status == HS_CLIENT_FETCH_HAVE_DESC)) {
+      /* This case is unique because it can NOT happen in theory. Once we get
+       * a new descriptor, the HS client subsystem is notified immediately and
+       * the connections waiting for it are handled which means the state will
+       * change from renddesc wait state. Log this and continue to next
+       * connection. */
+      continue;
+    }
+    /* In the case of an error, either all SOCKS connections have been
+     * closed or we are still missing directory information. Leave the
+     * connection in renddesc wait state so when we get more info, we'll be
+     * able to try it again. */
+  } SMARTLIST_FOREACH_END(base_conn);
+
+  /* We don't have ownership of those objects. */
+  smartlist_free(conns);
+}
+
+/* A v3 HS circuit successfully connected to the hidden service. Update the
+ * stream state at <b>hs_conn_ident</b> appropriately. */
+static void
+note_connection_attempt_succeeded(const hs_ident_edge_conn_t *hs_conn_ident)
+{
+  tor_assert(hs_conn_ident);
+
+  /* Remove from the hid serv cache all requests for that service so we can
+   * query the HSDir again later on for various reasons. */
+  purge_hid_serv_request(&hs_conn_ident->identity_pk);
+
+  /* The v2 subsystem cleans up the intro point time out flag at this stage.
+   * We don't try to do it here because we still need to keep intact the intro
+   * point state for future connections. Even though we are able to connect to
+   * the service, doesn't mean we should reset the timed out intro points.
+   *
+   * It is not possible to have successfully connected to an intro point
+   * present in our cache that was on error or timed out. Every entry in that
+   * cache have a 2 minutes lifetime so ultimately the intro point(s) state
+   * will be reset and thus possible to be retried. */
+}
+
+/* Given the pubkey of a hidden service in <b>onion_identity_pk</b>, fetch its
+ * descriptor by launching a dir connection to <b>hsdir</b>. Return a
+ * hs_client_fetch_status_t status code depending on how it went. */
+static hs_client_fetch_status_t
+directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
+                               const routerstatus_t *hsdir)
+{
+  uint64_t current_time_period = hs_get_time_period_num(0);
+  ed25519_public_key_t blinded_pubkey;
+  char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
+  hs_ident_dir_conn_t hs_conn_dir_ident;
+  int retval;
+
+  tor_assert(hsdir);
+  tor_assert(onion_identity_pk);
+
+  /* Get blinded pubkey */
+  hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
+                          current_time_period, &blinded_pubkey);
+  /* ...and base64 it. */
+  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
+  if (BUG(retval < 0)) {
+    return HS_CLIENT_FETCH_ERROR;
+  }
+
+  /* Copy onion pk to a dir_ident so that we attach it to the dir conn */
+  hs_ident_dir_conn_init(onion_identity_pk, &blinded_pubkey,
+                         &hs_conn_dir_ident);
+
+  /* Setup directory request */
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_HSDESC);
+  directory_request_set_routerstatus(req, hsdir);
+  directory_request_set_indirection(req, DIRIND_ANONYMOUS);
+  directory_request_set_resource(req, base64_blinded_pubkey);
+  directory_request_fetch_set_hs_ident(req, &hs_conn_dir_ident);
+  directory_initiate_request(req);
+  directory_request_free(req);
+
+  log_info(LD_REND, "Descriptor fetch request for service %s with blinded "
+                    "key %s to directory %s",
+           safe_str_client(ed25519_fmt(onion_identity_pk)),
+           safe_str_client(base64_blinded_pubkey),
+           safe_str_client(routerstatus_describe(hsdir)));
+
+  /* Fire a REQUESTED event on the control port. */
+  hs_control_desc_event_requested(onion_identity_pk, base64_blinded_pubkey,
+                                  hsdir);
+
+  /* Cleanup memory. */
+  memwipe(&blinded_pubkey, 0, sizeof(blinded_pubkey));
+  memwipe(base64_blinded_pubkey, 0, sizeof(base64_blinded_pubkey));
+  memwipe(&hs_conn_dir_ident, 0, sizeof(hs_conn_dir_ident));
+
+  return HS_CLIENT_FETCH_LAUNCHED;
+}
+
+/** Return the HSDir we should use to fetch the descriptor of the hidden
+ *  service with identity key <b>onion_identity_pk</b>. */
+STATIC routerstatus_t *
+pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
+{
+  int retval;
+  char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
+  uint64_t current_time_period = hs_get_time_period_num(0);
+  smartlist_t *responsible_hsdirs = NULL;
+  ed25519_public_key_t blinded_pubkey;
+  routerstatus_t *hsdir_rs = NULL;
+
+  tor_assert(onion_identity_pk);
+
+  /* Get blinded pubkey of hidden service */
+  hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
+                          current_time_period, &blinded_pubkey);
+  /* ...and base64 it. */
+  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
+  if (BUG(retval < 0)) {
+    return NULL;
+  }
+
+  /* Get responsible hsdirs of service for this time period */
+  responsible_hsdirs = smartlist_new();
+
+  hs_get_responsible_hsdirs(&blinded_pubkey, current_time_period,
+                            0, 1, responsible_hsdirs);
+
+  log_debug(LD_REND, "Found %d responsible HSDirs and about to pick one.",
+           smartlist_len(responsible_hsdirs));
+
+  /* Pick an HSDir from the responsible ones. The ownership of
+   * responsible_hsdirs is given to this function so no need to free it. */
+  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey);
+
+  return hsdir_rs;
+}
+
+/** Fetch a v3 descriptor using the given <b>onion_identity_pk</b>.
+ *
+ * On success, HS_CLIENT_FETCH_LAUNCHED is returned. Otherwise, an error from
+ * hs_client_fetch_status_t is returned. */
+MOCK_IMPL(STATIC hs_client_fetch_status_t,
+fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk))
+{
+  routerstatus_t *hsdir_rs =NULL;
+
+  tor_assert(onion_identity_pk);
+
+  hsdir_rs = pick_hsdir_v3(onion_identity_pk);
+  if (!hsdir_rs) {
+    log_info(LD_REND, "Couldn't pick a v3 hsdir.");
+    return HS_CLIENT_FETCH_NO_HSDIRS;
+  }
+
+  return directory_launch_v3_desc_fetch(onion_identity_pk, hsdir_rs);
+}
+
+/* Make sure that the given v3 origin circuit circ is a valid correct
+ * introduction circuit. This will BUG() on any problems and hard assert if
+ * the anonymity of the circuit is not ok. Return 0 on success else -1 where
+ * the circuit should be mark for closed immediately. */
+static int
+intro_circ_is_ok(const origin_circuit_t *circ)
+{
+  int ret = 0;
+
+  tor_assert(circ);
+
+  if (BUG(TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCING &&
+          TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT &&
+          TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) {
+    ret = -1;
+  }
+  if (BUG(circ->hs_ident == NULL)) {
+    ret = -1;
+  }
+  if (BUG(!hs_ident_intro_circ_is_valid(circ->hs_ident))) {
+    ret = -1;
+  }
+
+  /* This can stop the tor daemon but we want that since if we don't have
+   * anonymity on this circuit, something went really wrong. */
+  assert_circ_anonymity_ok(circ, get_options());
+  return ret;
+}
+
+/* Find a descriptor intro point object that matches the given ident in the
+ * given descriptor desc. Return NULL if not found. */
+static const hs_desc_intro_point_t *
+find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,
+                               const hs_descriptor_t *desc)
+{
+  const hs_desc_intro_point_t *intro_point = NULL;
+
+  tor_assert(ident);
+  tor_assert(desc);
+
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    if (ed25519_pubkey_eq(&ident->intro_auth_pk,
+                          &ip->auth_key_cert->signed_key)) {
+      intro_point = ip;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ip);
+
+  return intro_point;
+}
+
+/* Find a descriptor intro point object from the descriptor object desc that
+ * matches the given legacy identity digest in legacy_id. Return NULL if not
+ * found. */
+static hs_desc_intro_point_t *
+find_desc_intro_point_by_legacy_id(const char *legacy_id,
+                                   const hs_descriptor_t *desc)
+{
+  hs_desc_intro_point_t *ret_ip = NULL;
+
+  tor_assert(legacy_id);
+  tor_assert(desc);
+
+  /* We will go over every intro point and try to find which one is linked to
+   * that circuit. Those lists are small so it's not that expensive. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          hs_desc_intro_point_t *, ip) {
+    SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
+                            const hs_desc_link_specifier_t *, lspec) {
+      /* Not all tor node have an ed25519 identity key so we still rely on the
+       * legacy identity digest. */
+      if (lspec->type != LS_LEGACY_ID) {
+        continue;
+      }
+      if (fast_memneq(legacy_id, lspec->u.legacy_id, DIGEST_LEN)) {
+        break;
+      }
+      /* Found it. */
+      ret_ip = ip;
+      goto end;
+    } SMARTLIST_FOREACH_END(lspec);
+  } SMARTLIST_FOREACH_END(ip);
+
+ end:
+  return ret_ip;
+}
+
+/* Send an INTRODUCE1 cell along the intro circuit and populate the rend
+ * circuit identifier with the needed key material for the e2e encryption.
+ * Return 0 on success, -1 if there is a transient error such that an action
+ * has been taken to recover and -2 if there is a permanent error indicating
+ * that both circuits were closed. */
+static int
+send_introduce1(origin_circuit_t *intro_circ,
+                origin_circuit_t *rend_circ)
+{
+  int status;
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  const ed25519_public_key_t *service_identity_pk = NULL;
+  const hs_desc_intro_point_t *ip;
+
+  tor_assert(rend_circ);
+  if (intro_circ_is_ok(intro_circ) < 0) {
+    goto perm_err;
+  }
+
+  service_identity_pk = &intro_circ->hs_ident->identity_pk;
+  /* For logging purposes. There will be a time where the hs_ident will have a
+   * version number but for now there is none because it's all v3. */
+  hs_build_address(service_identity_pk, HS_VERSION_THREE, onion_address);
+
+  log_info(LD_REND, "Sending INTRODUCE1 cell to service %s on circuit %u",
+           safe_str_client(onion_address), TO_CIRCUIT(intro_circ)->n_circ_id);
+
+  /* 1) Get descriptor from our cache. */
+  const hs_descriptor_t *desc =
+    hs_cache_lookup_as_client(service_identity_pk);
+  if (desc == NULL || !hs_client_any_intro_points_usable(service_identity_pk,
+                                                         desc)) {
+    log_info(LD_REND, "Request to %s %s. Trying to fetch a new descriptor.",
+             safe_str_client(onion_address),
+             (desc) ? "didn't have usable intro points" :
+             "didn't have a descriptor");
+    hs_client_refetch_hsdesc(service_identity_pk);
+    /* We just triggered a refetch, make sure every connections are back
+     * waiting for that descriptor. */
+    flag_all_conn_wait_desc(service_identity_pk);
+    /* We just asked for a refetch so this is a transient error. */
+    goto tran_err;
+  }
+
+  /* We need to find which intro point in the descriptor we are connected to
+   * on intro_circ. */
+  ip = find_desc_intro_point_by_ident(intro_circ->hs_ident, desc);
+  if (BUG(ip == NULL)) {
+    /* If we can find a descriptor from this introduction circuit ident, we
+     * must have a valid intro point object. Permanent error. */
+    goto perm_err;
+  }
+
+  /* Send the INTRODUCE1 cell. */
+  if (hs_circ_send_introduce1(intro_circ, rend_circ, ip,
+                              desc->subcredential) < 0) {
+    /* Unable to send the cell, the intro circuit has been marked for close so
+     * this is a permanent error. */
+    tor_assert_nonfatal(TO_CIRCUIT(intro_circ)->marked_for_close);
+    goto perm_err;
+  }
+
+  /* Cell has been sent successfully. Copy the introduction point
+   * authentication and encryption key in the rendezvous circuit identifier so
+   * we can compute the ntor keys when we receive the RENDEZVOUS2 cell. */
+  memcpy(&rend_circ->hs_ident->intro_enc_pk, &ip->enc_key,
+         sizeof(rend_circ->hs_ident->intro_enc_pk));
+  ed25519_pubkey_copy(&rend_circ->hs_ident->intro_auth_pk,
+                      &intro_circ->hs_ident->intro_auth_pk);
+
+  /* Now, we wait for an ACK or NAK on this circuit. */
+  circuit_change_purpose(TO_CIRCUIT(intro_circ),
+                         CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT);
+  /* Set timestamp_dirty, because circuit_expire_building expects it to
+   * specify when a circuit entered the _C_INTRODUCE_ACK_WAIT state. */
+  TO_CIRCUIT(intro_circ)->timestamp_dirty = time(NULL);
+  pathbias_count_use_attempt(intro_circ);
+
+  /* Success. */
+  status = 0;
+  goto end;
+
+ perm_err:
+  /* Permanent error: it is possible that the intro circuit was closed prior
+   * because we weren't able to send the cell. Make sure we don't double close
+   * it which would result in a warning. */
+  if (!TO_CIRCUIT(intro_circ)->marked_for_close) {
+    circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_INTERNAL);
+  }
+  circuit_mark_for_close(TO_CIRCUIT(rend_circ), END_CIRC_REASON_INTERNAL);
+  status = -2;
+  goto end;
+
+ tran_err:
+  status = -1;
+
+ end:
+  memwipe(onion_address, 0, sizeof(onion_address));
+  return status;
+}
+
+/* Using the introduction circuit circ, setup the authentication key of the
+ * intro point this circuit has extended to. */
+static void
+setup_intro_circ_auth_key(origin_circuit_t *circ)
+{
+  const hs_descriptor_t *desc;
+  const hs_desc_intro_point_t *ip;
+
+  tor_assert(circ);
+
+  desc = hs_cache_lookup_as_client(&circ->hs_ident->identity_pk);
+  if (BUG(desc == NULL)) {
+    /* Opening intro circuit without the descriptor is no good... */
+    goto end;
+  }
+
+  /* We will go over every intro point and try to find which one is linked to
+   * that circuit. Those lists are small so it's not that expensive. */
+  ip = find_desc_intro_point_by_legacy_id(
+                       circ->build_state->chosen_exit->identity_digest, desc);
+  if (ip) {
+    /* We got it, copy its authentication key to the identifier. */
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+    goto end;
+  }
+
+  /* Reaching this point means we didn't find any intro point for this circuit
+   * which is not suppose to happen. */
+  tor_assert_nonfatal_unreached();
+
+ end:
+  return;
+}
+
+/* Called when an introduction circuit has opened. */
+static void
+client_intro_circ_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
+  log_info(LD_REND, "Introduction circuit %u has opened. Attaching streams.",
+           (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+
+  /* This is an introduction circuit so we'll attach the correct
+   * authentication key to the circuit identifier so it can be identified
+   * properly later on. */
+  setup_intro_circ_auth_key(circ);
+
+  connection_ap_attach_pending(1);
+}
+
+/* Called when a rendezvous circuit has opened. */
+static void
+client_rendezvous_circ_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+
+  const extend_info_t *rp_ei = circ->build_state->chosen_exit;
+
+  /* Check that we didn't accidentally choose a node that does not understand
+   * the v3 rendezvous protocol */
+  if (rp_ei) {
+    const node_t *rp_node = node_get_by_id(rp_ei->identity_digest);
+    if (rp_node) {
+      if (BUG(!node_supports_v3_rendezvous_point(rp_node))) {
+        return;
+      }
+    }
+  }
+
+  log_info(LD_REND, "Rendezvous circuit has opened to %s.",
+           safe_str_client(extend_info_describe(rp_ei)));
+
+  /* Ignore returned value, nothing we can really do. On failure, the circuit
+   * will be marked for close. */
+  hs_circ_send_establish_rendezvous(circ);
+
+  /* Register rend circuit in circuitmap if it's still alive. */
+  if (!TO_CIRCUIT(circ)->marked_for_close) {
+    hs_circuitmap_register_rend_circ_client_side(circ,
+                                     circ->hs_ident->rendezvous_cookie);
+  }
+}
+
+/* This is an helper function that convert a descriptor intro point object ip
+ * to a newly allocated extend_info_t object fully initialized. Return NULL if
+ * we can't convert it for which chances are that we are missing or malformed
+ * link specifiers. */
+STATIC extend_info_t *
+desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip)
+{
+  extend_info_t *ei;
+  smartlist_t *lspecs = smartlist_new();
+
+  tor_assert(ip);
+
+  /* We first encode the descriptor link specifiers into the binary
+   * representation which is a trunnel object. */
+  SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
+                          const hs_desc_link_specifier_t *, desc_lspec) {
+    link_specifier_t *lspec = hs_desc_lspec_to_trunnel(desc_lspec);
+    smartlist_add(lspecs, lspec);
+  } SMARTLIST_FOREACH_END(desc_lspec);
+
+  /* Explicitly put the direct connection option to 0 because this is client
+   * side and there is no such thing as a non anonymous client. */
+  ei = hs_get_extend_info_from_lspecs(lspecs, &ip->onion_key, 0);
+
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls, link_specifier_free(ls));
+  smartlist_free(lspecs);
+  return ei;
+}
+
+/* Return true iff the intro point ip for the service service_pk is usable.
+ * This function checks if the intro point is in the client intro state cache
+ * and checks at the failures. It is considered usable if:
+ *   - No error happened (INTRO_POINT_FAILURE_GENERIC)
+ *   - It is not flagged as timed out (INTRO_POINT_FAILURE_TIMEOUT)
+ *   - The unreachable count is lower than
+ *     MAX_INTRO_POINT_REACHABILITY_FAILURES (INTRO_POINT_FAILURE_UNREACHABLE)
+ */
+static int
+intro_point_is_usable(const ed25519_public_key_t *service_pk,
+                      const hs_desc_intro_point_t *ip)
+{
+  const hs_cache_intro_state_t *state;
+
+  tor_assert(service_pk);
+  tor_assert(ip);
+
+  state = hs_cache_client_intro_state_find(service_pk,
+                                           &ip->auth_key_cert->signed_key);
+  if (state == NULL) {
+    /* This means we've never encountered any problem thus usable. */
+    goto usable;
+  }
+  if (state->error) {
+    log_info(LD_REND, "Intro point with auth key %s had an error. Not usable",
+             safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)));
+    goto not_usable;
+  }
+  if (state->timed_out) {
+    log_info(LD_REND, "Intro point with auth key %s timed out. Not usable",
+             safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)));
+    goto not_usable;
+  }
+  if (state->unreachable_count >= MAX_INTRO_POINT_REACHABILITY_FAILURES) {
+    log_info(LD_REND, "Intro point with auth key %s unreachable. Not usable",
+             safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)));
+    goto not_usable;
+  }
+
+ usable:
+  return 1;
+ not_usable:
+  return 0;
+}
+
+/* Using a descriptor desc, return a newly allocated extend_info_t object of a
+ * randomly picked introduction point from its list. Return NULL if none are
+ * usable. */
+STATIC extend_info_t *
+client_get_random_intro(const ed25519_public_key_t *service_pk)
+{
+  extend_info_t *ei = NULL, *ei_excluded = NULL;
+  smartlist_t *usable_ips = NULL;
+  const hs_descriptor_t *desc;
+  const hs_desc_encrypted_data_t *enc_data;
+  const or_options_t *options = get_options();
+  /* Calculate the onion address for logging purposes */
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  tor_assert(service_pk);
+
+  desc = hs_cache_lookup_as_client(service_pk);
+  /* Assume the service is v3 if the descriptor is missing. This is ok,
+   * because we only use the address in log messages */
+  hs_build_address(service_pk,
+                   desc ? desc->plaintext_data.version : HS_VERSION_THREE,
+                   onion_address);
+  if (desc == NULL || !hs_client_any_intro_points_usable(service_pk,
+                                                         desc)) {
+    log_info(LD_REND, "Unable to randomly select an introduction point "
+             "for service %s because descriptor %s. We can't connect.",
+             safe_str_client(onion_address),
+             (desc) ? "doesn't have any usable intro points"
+                    : "is missing (assuming v3 onion address)");
+    goto end;
+  }
+
+  enc_data = &desc->encrypted_data;
+  usable_ips = smartlist_new();
+  smartlist_add_all(usable_ips, enc_data->intro_points);
+  while (smartlist_len(usable_ips) != 0) {
+    int idx;
+    const hs_desc_intro_point_t *ip;
+
+    /* Pick a random intro point and immediately remove it from the usable
+     * list so we don't pick it again if we have to iterate more. */
+    idx = crypto_rand_int(smartlist_len(usable_ips));
+    ip = smartlist_get(usable_ips, idx);
+    smartlist_del(usable_ips, idx);
+
+    /* We need to make sure we have a usable intro points which is in a good
+     * state in our cache. */
+    if (!intro_point_is_usable(service_pk, ip)) {
+      continue;
+    }
+
+    /* Generate an extend info object from the intro point object. */
+    ei = desc_intro_point_to_extend_info(ip);
+    if (ei == NULL) {
+      /* We can get here for instance if the intro point is a private address
+       * and we aren't allowed to extend to those. */
+      log_info(LD_REND, "Unable to select introduction point with auth key %s "
+               "for service %s, because we could not extend to it.",
+               safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)),
+               safe_str_client(onion_address));
+      continue;
+    }
+
+    /* Test the pick against ExcludeNodes. */
+    if (routerset_contains_extendinfo(options->ExcludeNodes, ei)) {
+      /* If this pick is in the ExcludeNodes list, we keep its reference so if
+       * we ever end up not being able to pick anything else and StrictNodes is
+       * unset, we'll use it. */
+      if (ei_excluded) {
+        /* If something was already here free it. After the loop is gone we
+         * will examine the last excluded intro point, and that's fine since
+         * that's random anyway */
+        extend_info_free(ei_excluded);
+      }
+      ei_excluded = ei;
+      continue;
+    }
+
+    /* Good pick! Let's go with this. */
+    goto end;
+  }
+
+  /* Reaching this point means a couple of things. Either we can't use any of
+   * the intro point listed because the IP address can't be extended to or it
+   * is listed in the ExcludeNodes list. In the later case, if StrictNodes is
+   * set, we are forced to not use anything. */
+  ei = ei_excluded;
+  if (options->StrictNodes) {
+    log_warn(LD_REND, "Every introduction point for service %s is in the "
+             "ExcludeNodes set and StrictNodes is set. We can't connect.",
+             safe_str_client(onion_address));
+    extend_info_free(ei);
+    ei = NULL;
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND, "Every introduction point for service "
+           "%s is unusable or we can't extend to it. We can't connect.",
+           safe_str_client(onion_address));
+  }
+
+ end:
+  smartlist_free(usable_ips);
+  memwipe(onion_address, 0, sizeof(onion_address));
+  return ei;
+}
+
+/* For this introduction circuit, we'll look at if we have any usable
+ * introduction point left for this service. If so, we'll use the circuit to
+ * re-extend to a new intro point. Else, we'll close the circuit and its
+ * corresponding rendezvous circuit. Return 0 if we are re-extending else -1
+ * if we are closing the circuits.
+ *
+ * This is called when getting an INTRODUCE_ACK cell with a NACK. */
+static int
+close_or_reextend_intro_circ(origin_circuit_t *intro_circ)
+{
+  int ret = -1;
+  const hs_descriptor_t *desc;
+  origin_circuit_t *rend_circ;
+
+  tor_assert(intro_circ);
+
+  desc = hs_cache_lookup_as_client(&intro_circ->hs_ident->identity_pk);
+  if (BUG(desc == NULL)) {
+    /* We can't continue without a descriptor. */
+    goto close;
+  }
+  /* We still have the descriptor, great! Let's try to see if we can
+   * re-extend by looking up if there are any usable intro points. */
+  if (!hs_client_any_intro_points_usable(&intro_circ->hs_ident->identity_pk,
+                                         desc)) {
+    goto close;
+  }
+  /* Try to re-extend now. */
+  if (hs_client_reextend_intro_circuit(intro_circ) < 0) {
+    goto close;
+  }
+  /* Success on re-extending. Don't return an error. */
+  ret = 0;
+  goto end;
+
+ close:
+  /* Change the intro circuit purpose before so we don't report an intro point
+   * failure again triggering an extra descriptor fetch. The circuit can
+   * already be closed on failure to re-extend. */
+  if (!TO_CIRCUIT(intro_circ)->marked_for_close) {
+    circuit_change_purpose(TO_CIRCUIT(intro_circ),
+                           CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
+    circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_FINISHED);
+  }
+  /* Close the related rendezvous circuit. */
+  rend_circ = hs_circuitmap_get_rend_circ_client_side(
+                                     intro_circ->hs_ident->rendezvous_cookie);
+  /* The rendezvous circuit might have collapsed while the INTRODUCE_ACK was
+   * inflight so we can't expect one every time. */
+  if (rend_circ) {
+    circuit_mark_for_close(TO_CIRCUIT(rend_circ), END_CIRC_REASON_FINISHED);
+  }
+
+ end:
+  return ret;
+}
+
+/* Called when we get an INTRODUCE_ACK success status code. Do the appropriate
+ * actions for the rendezvous point and finally close intro_circ. */
+static void
+handle_introduce_ack_success(origin_circuit_t *intro_circ)
+{
+  origin_circuit_t *rend_circ = NULL;
+
+  tor_assert(intro_circ);
+
+  log_info(LD_REND, "Received INTRODUCE_ACK ack! Informing rendezvous");
+
+  /* Get the rendezvous circuit for this rendezvous cookie. */
+  uint8_t *rendezvous_cookie = intro_circ->hs_ident->rendezvous_cookie;
+  rend_circ =
+  hs_circuitmap_get_established_rend_circ_client_side(rendezvous_cookie);
+  if (rend_circ == NULL) {
+    log_warn(LD_REND, "Can't find any rendezvous circuit. Stopping");
+    goto end;
+  }
+
+  assert_circ_anonymity_ok(rend_circ, get_options());
+
+  /* It is possible to get a RENDEZVOUS2 cell before the INTRODUCE_ACK which
+   * means that the circuit will be joined and already transmitting data. In
+   * that case, simply skip the purpose change and close the intro circuit
+   * like it should be. */
+  if (TO_CIRCUIT(rend_circ)->purpose == CIRCUIT_PURPOSE_C_REND_JOINED) {
+    goto end;
+  }
+  circuit_change_purpose(TO_CIRCUIT(rend_circ),
+                         CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+  /* Set timestamp_dirty, because circuit_expire_building expects it to
+   * specify when a circuit entered the
+   * CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED state. */
+  TO_CIRCUIT(rend_circ)->timestamp_dirty = time(NULL);
+
+ end:
+  /* We don't need the intro circuit anymore. It did what it had to do! */
+  circuit_change_purpose(TO_CIRCUIT(intro_circ),
+                         CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
+  circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_FINISHED);
+
+  /* XXX: Close pending intro circuits we might have in parallel. */
+  return;
+}
+
+/* Called when we get an INTRODUCE_ACK failure status code. Depending on our
+ * failure cache status, either close the circuit or re-extend to a new
+ * introduction point. */
+static void
+handle_introduce_ack_bad(origin_circuit_t *circ, int status)
+{
+  tor_assert(circ);
+
+  log_info(LD_REND, "Received INTRODUCE_ACK nack by %s. Reason: %u",
+      safe_str_client(extend_info_describe(circ->build_state->chosen_exit)),
+      status);
+
+  /* It's a NAK. The introduction point didn't relay our request. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
+
+  /* Note down this failure in the intro point failure cache. Depending on how
+   * many times we've tried this intro point, close it or reextend. */
+  hs_cache_client_intro_state_note(&circ->hs_ident->identity_pk,
+                                   &circ->hs_ident->intro_auth_pk,
+                                   INTRO_POINT_FAILURE_GENERIC);
+}
+
+/* Called when we get an INTRODUCE_ACK on the intro circuit circ. The encoded
+ * cell is in payload of length payload_len. Return 0 on success else a
+ * negative value. The circuit is either close or reuse to re-extend to a new
+ * introduction point. */
+static int
+handle_introduce_ack(origin_circuit_t *circ, const uint8_t *payload,
+                     size_t payload_len)
+{
+  int status, ret = -1;
+
+  tor_assert(circ);
+  tor_assert(circ->build_state);
+  tor_assert(circ->build_state->chosen_exit);
+  assert_circ_anonymity_ok(circ, get_options());
+  tor_assert(payload);
+
+  status = hs_cell_parse_introduce_ack(payload, payload_len);
+  switch (status) {
+  case HS_CELL_INTRO_ACK_SUCCESS:
+    ret = 0;
+    handle_introduce_ack_success(circ);
+    goto end;
+  case HS_CELL_INTRO_ACK_FAILURE:
+  case HS_CELL_INTRO_ACK_BADFMT:
+  case HS_CELL_INTRO_ACK_NORELAY:
+    handle_introduce_ack_bad(circ, status);
+    /* We are going to see if we have to close the circuits (IP and RP) or we
+     * can re-extend to a new intro point. */
+    ret = close_or_reextend_intro_circ(circ);
+    break;
+  default:
+    log_info(LD_PROTOCOL, "Unknown INTRODUCE_ACK status code %u from %s",
+        status,
+        safe_str_client(extend_info_describe(circ->build_state->chosen_exit)));
+    break;
+  }
+
+ end:
+  return ret;
+}
+
+/* Called when we get a RENDEZVOUS2 cell on the rendezvous circuit circ. The
+ * encoded cell is in payload of length payload_len. Return 0 on success or a
+ * negative value on error. On error, the circuit is marked for close. */
+STATIC int
+handle_rendezvous2(origin_circuit_t *circ, const uint8_t *payload,
+                   size_t payload_len)
+{
+  int ret = -1;
+  curve25519_public_key_t server_pk;
+  uint8_t auth_mac[DIGEST256_LEN] = {0};
+  uint8_t handshake_info[CURVE25519_PUBKEY_LEN + sizeof(auth_mac)] = {0};
+  hs_ntor_rend_cell_keys_t keys;
+  const hs_ident_circuit_t *ident;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  /* Make things easier. */
+  ident = circ->hs_ident;
+  tor_assert(ident);
+
+  if (hs_cell_parse_rendezvous2(payload, payload_len, handshake_info,
+                                sizeof(handshake_info)) < 0) {
+    goto err;
+  }
+  /* Get from the handshake info the SERVER_PK and AUTH_MAC. */
+  memcpy(&server_pk, handshake_info, CURVE25519_PUBKEY_LEN);
+  memcpy(auth_mac, handshake_info + CURVE25519_PUBKEY_LEN, sizeof(auth_mac));
+
+  /* Generate the handshake info. */
+  if (hs_ntor_client_get_rendezvous1_keys(&ident->intro_auth_pk,
+                                          &ident->rendezvous_client_kp,
+                                          &ident->intro_enc_pk, &server_pk,
+                                          &keys) < 0) {
+    log_info(LD_REND, "Unable to compute the rendezvous keys.");
+    goto err;
+  }
+
+  /* Critical check, make sure that the MAC matches what we got with what we
+   * computed just above. */
+  if (!hs_ntor_client_rendezvous2_mac_is_good(&keys, auth_mac)) {
+    log_info(LD_REND, "Invalid MAC in RENDEZVOUS2. Rejecting cell.");
+    goto err;
+  }
+
+  /* Setup the e2e encryption on the circuit and finalize its state. */
+  if (hs_circuit_setup_e2e_rend_circ(circ, keys.ntor_key_seed,
+                                     sizeof(keys.ntor_key_seed), 0) < 0) {
+    log_info(LD_REND, "Unable to setup the e2e encryption.");
+    goto err;
+  }
+  /* Success. Hidden service connection finalized! */
+  ret = 0;
+  goto end;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+ end:
+  memwipe(&keys, 0, sizeof(keys));
+  return ret;
+}
+
+/* Return true iff the client can fetch a descriptor for this service public
+ * identity key and status_out if not NULL is untouched. If the client can
+ * _not_ fetch the descriptor and if status_out is not NULL, it is set with
+ * the fetch status code. */
+static unsigned int
+can_client_refetch_desc(const ed25519_public_key_t *identity_pk,
+                        hs_client_fetch_status_t *status_out)
+{
+  hs_client_fetch_status_t status;
+
+  tor_assert(identity_pk);
+
+  /* Are we configured to fetch descriptors? */
+  if (!get_options()->FetchHidServDescriptors) {
+    log_warn(LD_REND, "We received an onion address for a hidden service "
+                      "descriptor but we are configured to not fetch.");
+    status = HS_CLIENT_FETCH_NOT_ALLOWED;
+    goto cannot;
+  }
+
+  /* Without a live consensus we can't do any client actions. It is needed to
+   * compute the hashring for a service. */
+  if (!networkstatus_get_live_consensus(approx_time())) {
+    log_info(LD_REND, "Can't fetch descriptor for service %s because we "
+                      "are missing a live consensus. Stalling connection.",
+             safe_str_client(ed25519_fmt(identity_pk)));
+    status = HS_CLIENT_FETCH_MISSING_INFO;
+    goto cannot;
+  }
+
+  if (!router_have_minimum_dir_info()) {
+    log_info(LD_REND, "Can't fetch descriptor for service %s because we "
+                      "dont have enough descriptors. Stalling connection.",
+             safe_str_client(ed25519_fmt(identity_pk)));
+    status = HS_CLIENT_FETCH_MISSING_INFO;
+    goto cannot;
+  }
+
+  /* Check if fetching a desc for this HS is useful to us right now */
+  {
+    const hs_descriptor_t *cached_desc = NULL;
+    cached_desc = hs_cache_lookup_as_client(identity_pk);
+    if (cached_desc && hs_client_any_intro_points_usable(identity_pk,
+                                                         cached_desc)) {
+      log_info(LD_GENERAL, "We would fetch a v3 hidden service descriptor "
+                           "but we already have a usable descriptor.");
+      status = HS_CLIENT_FETCH_HAVE_DESC;
+      goto cannot;
+    }
+  }
+
+  /* Don't try to refetch while we have a pending request for it. */
+  if (directory_request_is_pending(identity_pk)) {
+    log_info(LD_REND, "Already a pending directory request. Waiting on it.");
+    status = HS_CLIENT_FETCH_PENDING;
+    goto cannot;
+  }
+
+  /* Yes, client can fetch! */
+  return 1;
+ cannot:
+  if (status_out) {
+    *status_out = status;
+  }
+  return 0;
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/** A circuit just finished connecting to a hidden service that the stream
+ *  <b>conn</b> has been waiting for. Let the HS subsystem know about this. */
+void
+hs_client_note_connection_attempt_succeeded(const edge_connection_t *conn)
+{
+  tor_assert(connection_edge_is_rendezvous_stream(conn));
+
+  if (BUG(conn->rend_data && conn->hs_ident)) {
+    log_warn(LD_BUG, "Stream had both rend_data and hs_ident..."
+             "Prioritizing hs_ident");
+  }
+
+  if (conn->hs_ident) { /* It's v3: pass it to the prop224 handler */
+    note_connection_attempt_succeeded(conn->hs_ident);
+    return;
+  } else if (conn->rend_data) { /* It's v2: pass it to the legacy handler */
+    rend_client_note_connection_attempt_ended(conn->rend_data);
+    return;
+  }
+}
+
+/* With the given encoded descriptor in desc_str and the service key in
+ * service_identity_pk, decode the descriptor and set the desc pointer with a
+ * newly allocated descriptor object.
+ *
+ * Return 0 on success else a negative value and desc is set to NULL. */
+int
+hs_client_decode_descriptor(const char *desc_str,
+                            const ed25519_public_key_t *service_identity_pk,
+                            hs_descriptor_t **desc)
+{
+  int ret;
+  uint8_t subcredential[DIGEST256_LEN];
+  ed25519_public_key_t blinded_pubkey;
+
+  tor_assert(desc_str);
+  tor_assert(service_identity_pk);
+  tor_assert(desc);
+
+  /* Create subcredential for this HS so that we can decrypt */
+  {
+    uint64_t current_time_period = hs_get_time_period_num(0);
+    hs_build_blinded_pubkey(service_identity_pk, NULL, 0, current_time_period,
+                            &blinded_pubkey);
+    hs_get_subcredential(service_identity_pk, &blinded_pubkey, subcredential);
+  }
+
+  /* Parse descriptor */
+  ret = hs_desc_decode_descriptor(desc_str, subcredential, desc);
+  memwipe(subcredential, 0, sizeof(subcredential));
+  if (ret < 0) {
+    log_warn(LD_GENERAL, "Could not parse received descriptor as client.");
+    if (get_options()->SafeLogging_ == SAFELOG_SCRUB_NONE) {
+      log_warn(LD_GENERAL, "%s", escaped(desc_str));
+    }
+    goto err;
+  }
+
+  /* Make sure the descriptor signing key cross certifies with the computed
+   * blinded key. Without this validation, anyone knowing the subcredential
+   * and onion address can forge a descriptor. */
+  tor_cert_t *cert = (*desc)->plaintext_data.signing_key_cert;
+  if (tor_cert_checksig(cert,
+                        &blinded_pubkey, approx_time()) < 0) {
+    log_warn(LD_GENERAL, "Descriptor signing key certificate signature "
+             "doesn't validate with computed blinded key: %s",
+             tor_cert_describe_signature_status(cert));
+    goto err;
+  }
+
+  return 0;
+ err:
+  return -1;
+}
+
+/* Return true iff there are at least one usable intro point in the service
+ * descriptor desc. */
+int
+hs_client_any_intro_points_usable(const ed25519_public_key_t *service_pk,
+                                  const hs_descriptor_t *desc)
+{
+  tor_assert(service_pk);
+  tor_assert(desc);
+
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    if (intro_point_is_usable(service_pk, ip)) {
+      goto usable;
+    }
+  } SMARTLIST_FOREACH_END(ip);
+
+  return 0;
+ usable:
+  return 1;
+}
+
+/** Launch a connection to a hidden service directory to fetch a hidden
+ * service descriptor using <b>identity_pk</b> to get the necessary keys.
+ *
+ * A hs_client_fetch_status_t code is returned. */
+int
+hs_client_refetch_hsdesc(const ed25519_public_key_t *identity_pk)
+{
+  hs_client_fetch_status_t status;
+
+  tor_assert(identity_pk);
+
+  if (!can_client_refetch_desc(identity_pk, &status)) {
+    return status;
+  }
+
+  /* Try to fetch the desc and if we encounter an unrecoverable error, mark
+   * the desc as unavailable for now. */
+  status = fetch_v3_desc(identity_pk);
+  if (fetch_status_should_close_socks(status)) {
+    close_all_socks_conns_waiting_for_desc(identity_pk, status,
+                                           END_STREAM_REASON_RESOLVEFAILED);
+    /* Remove HSDir fetch attempts so that we can retry later if the user
+     * wants us to regardless of if we closed any connections. */
+    purge_hid_serv_request(identity_pk);
+  }
+  return status;
+}
+
+/* This is called when we are trying to attach an AP connection to these
+ * hidden service circuits from connection_ap_handshake_attach_circuit().
+ * Return 0 on success, -1 for a transient error that is actions were
+ * triggered to recover or -2 for a permenent error where both circuits will
+ * marked for close.
+ *
+ * The following supports every hidden service version. */
+int
+hs_client_send_introduce1(origin_circuit_t *intro_circ,
+                          origin_circuit_t *rend_circ)
+{
+  return (intro_circ->hs_ident) ? send_introduce1(intro_circ, rend_circ) :
+                                  rend_client_send_introduction(intro_circ,
+                                                                rend_circ);
+}
+
+/* Called when the client circuit circ has been established. It can be either
+ * an introduction or rendezvous circuit. This function handles all hidden
+ * service versions. */
+void
+hs_client_circuit_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Handle both version. v2 uses rend_data and v3 uses the hs circuit
+   * identifier hs_ident. Can't be both. */
+  switch (TO_CIRCUIT(circ)->purpose) {
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+    if (circ->hs_ident) {
+      client_intro_circ_has_opened(circ);
+    } else {
+      rend_client_introcirc_has_opened(circ);
+    }
+    break;
+  case CIRCUIT_PURPOSE_C_ESTABLISH_REND:
+    if (circ->hs_ident) {
+      client_rendezvous_circ_has_opened(circ);
+    } else {
+      rend_client_rendcirc_has_opened(circ);
+    }
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+  }
+}
+
+/* Called when we receive a RENDEZVOUS_ESTABLISHED cell. Change the state of
+ * the circuit to CIRCUIT_PURPOSE_C_REND_READY. Return 0 on success else a
+ * negative value and the circuit marked for close. */
+int
+hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
+                                   const uint8_t *payload, size_t payload_len)
+{
+  tor_assert(circ);
+  tor_assert(payload);
+
+  (void) payload_len;
+
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_ESTABLISH_REND) {
+    log_warn(LD_PROTOCOL, "Got a RENDEZVOUS_ESTABLISHED but we were not "
+                          "expecting one. Closing circuit.");
+    goto err;
+  }
+
+  log_info(LD_REND, "Received an RENDEZVOUS_ESTABLISHED. This circuit is "
+                    "now ready for rendezvous.");
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_REND_READY);
+
+  /* Set timestamp_dirty, because circuit_expire_building expects it to
+   * specify when a circuit entered the _C_REND_READY state. */
+  TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+
+  /* From a path bias point of view, this circuit is now successfully used.
+   * Waiting any longer opens us up to attacks from malicious hidden services.
+   * They could induce the client to attempt to connect to their hidden
+   * service and never reply to the client's rend requests */
+  pathbias_mark_use_success(circ);
+
+  /* If we already have the introduction circuit built, make sure we send
+   * the INTRODUCE cell _now_ */
+  connection_ap_attach_pending(1);
+
+  return 0;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* This is called when a descriptor has arrived following a fetch request and
+ * has been stored in the client cache. Every entry connection that matches
+ * the service identity key in the ident will get attached to the hidden
+ * service circuit. */
+void
+hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident)
+{
+  time_t now = time(NULL);
+  smartlist_t *conns = NULL;
+
+  tor_assert(ident);
+
+  conns = connection_list_by_type_state(CONN_TYPE_AP,
+                                        AP_CONN_STATE_RENDDESC_WAIT);
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    const hs_descriptor_t *desc;
+    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
+    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+
+    /* Only consider the entry connections that matches the service for which
+     * we just fetched its descriptor. */
+    if (!edge_conn->hs_ident ||
+        !ed25519_pubkey_eq(&ident->identity_pk,
+                           &edge_conn->hs_ident->identity_pk)) {
+      continue;
+    }
+    assert_connection_ok(base_conn, now);
+
+    /* We were just called because we stored the descriptor for this service
+     * so not finding a descriptor means we have a bigger problem. */
+    desc = hs_cache_lookup_as_client(&ident->identity_pk);
+    if (BUG(desc == NULL)) {
+      goto end;
+    }
+
+    if (!hs_client_any_intro_points_usable(&ident->identity_pk, desc)) {
+      log_info(LD_REND, "Hidden service descriptor is unusable. "
+                        "Closing streams.");
+      connection_mark_unattached_ap(entry_conn,
+                                    END_STREAM_REASON_RESOLVEFAILED);
+      /* We are unable to use the descriptor so remove the directory request
+       * from the cache so the next connection can try again. */
+      note_connection_attempt_succeeded(edge_conn->hs_ident);
+      goto end;
+    }
+
+    log_info(LD_REND, "Descriptor has arrived. Launching circuits.");
+
+    /* Because the connection can now proceed to opening circuit and
+     * ultimately connect to the service, reset those timestamp so the
+     * connection is considered "fresh" and can continue without being closed
+     * too early. */
+    base_conn->timestamp_created = now;
+    base_conn->timestamp_last_read_allowed = now;
+    base_conn->timestamp_last_write_allowed = now;
+    /* Change connection's state into waiting for a circuit. */
+    base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT;
+
+    connection_ap_mark_as_pending_circuit(entry_conn);
+  } SMARTLIST_FOREACH_END(base_conn);
+
+ end:
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(conns);
+}
+
+/* Return a newly allocated extend_info_t for a randomly chosen introduction
+ * point for the given edge connection identifier ident. Return NULL if we
+ * can't pick any usable introduction points. */
+extend_info_t *
+hs_client_get_random_intro_from_edge(const edge_connection_t *edge_conn)
+{
+  tor_assert(edge_conn);
+
+  return (edge_conn->hs_ident) ?
+    client_get_random_intro(&edge_conn->hs_ident->identity_pk) :
+    rend_client_get_random_intro(edge_conn->rend_data);
+}
+/* Called when get an INTRODUCE_ACK cell on the introduction circuit circ.
+ * Return 0 on success else a negative value is returned. The circuit will be
+ * closed or reuse to extend again to another intro point. */
+int
+hs_client_receive_introduce_ack(origin_circuit_t *circ,
+                                const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
+    log_warn(LD_PROTOCOL, "Unexpected INTRODUCE_ACK on circuit %u.",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+    goto end;
+  }
+
+  ret = (circ->hs_ident) ? handle_introduce_ack(circ, payload, payload_len) :
+                           rend_client_introduction_acked(circ, payload,
+                                                          payload_len);
+  /* For path bias: This circuit was used successfully. NACK or ACK counts. */
+  pathbias_mark_use_success(circ);
+
+ end:
+  return ret;
+}
+
+/* Called when get a RENDEZVOUS2 cell on the rendezvous circuit circ.  Return
+ * 0 on success else a negative value is returned. The circuit will be closed
+ * on error. */
+int
+hs_client_receive_rendezvous2(origin_circuit_t *circ,
+                              const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  /* Circuit can possibly be in both state because we could receive a
+   * RENDEZVOUS2 cell before the INTRODUCE_ACK has been received. */
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_REND_READY &&
+      TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) {
+    log_warn(LD_PROTOCOL, "Unexpected RENDEZVOUS2 cell on circuit %u. "
+                          "Closing circuit.",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+    goto end;
+  }
+
+  log_info(LD_REND, "Got RENDEZVOUS2 cell from hidden service on circuit %u.",
+           TO_CIRCUIT(circ)->n_circ_id);
+
+  ret = (circ->hs_ident) ? handle_rendezvous2(circ, payload, payload_len) :
+                           rend_client_receive_rendezvous(circ, payload,
+                                                          payload_len);
+ end:
+  return ret;
+}
+
+/* Extend the introduction circuit circ to another valid introduction point
+ * for the hidden service it is trying to connect to, or mark it and launch a
+ * new circuit if we can't extend it.  Return 0 on success or possible
+ * success. Return -1 and mark the introduction circuit for close on permanent
+ * failure.
+ *
+ * On failure, the caller is responsible for marking the associated rendezvous
+ * circuit for close. */
+int
+hs_client_reextend_intro_circuit(origin_circuit_t *circ)
+{
+  int ret = -1;
+  extend_info_t *ei;
+
+  tor_assert(circ);
+
+  ei = (circ->hs_ident) ?
+    client_get_random_intro(&circ->hs_ident->identity_pk) :
+    rend_client_get_random_intro(circ->rend_data);
+  if (ei == NULL) {
+    log_warn(LD_REND, "No usable introduction points left. Closing.");
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+    goto end;
+  }
+
+  if (circ->remaining_relay_early_cells) {
+    log_info(LD_REND, "Re-extending circ %u, this time to %s.",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(extend_info_describe(ei)));
+    ret = circuit_extend_to_new_exit(circ, ei);
+    if (ret == 0) {
+      /* We were able to extend so update the timestamp so we avoid expiring
+       * this circuit too early. The intro circuit is short live so the
+       * linkability issue is minimized, we just need the circuit to hold a
+       * bit longer so we can introduce. */
+      TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+    }
+  } else {
+    log_info(LD_REND, "Closing intro circ %u (out of RELAY_EARLY cells).",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
+    /* connection_ap_handshake_attach_circuit will launch a new intro circ. */
+    ret = 0;
+  }
+
+ end:
+  extend_info_free(ei);
+  return ret;
+}
+
+/* Release all the storage held by the client subsystem. */
+void
+hs_client_free_all(void)
+{
+  /* Purge the hidden service request cache. */
+  hs_purge_last_hid_serv_requests();
+}
+
+/* Purge all potentially remotely-detectable state held in the hidden
+ * service client code. Called on SIGNAL NEWNYM. */
+void
+hs_client_purge_state(void)
+{
+  /* v2 subsystem. */
+  rend_client_purge_state();
+
+  /* Cancel all descriptor fetches. Do this first so once done we are sure
+   * that our descriptor cache won't modified. */
+  cancel_descriptor_fetches();
+  /* Purge the introduction point state cache. */
+  hs_cache_client_intro_state_purge();
+  /* Purge the descriptor cache. */
+  hs_cache_purge_as_client();
+  /* Purge the last hidden service request cache. */
+  hs_purge_last_hid_serv_requests();
+
+  log_info(LD_REND, "Hidden service client state has been purged.");
+}
+
+/* Called when our directory information has changed. */
+void
+hs_client_dir_info_changed(void)
+{
+  /* We have possibly reached the minimum directory information or new
+   * consensus so retry all pending SOCKS connection in
+   * AP_CONN_STATE_RENDDESC_WAIT state in order to fetch the descriptor. */
+  retry_all_socks_conn_waiting_for_desc();
+}
diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h
new file mode 100644
index 0000000000..8083910747
--- /dev/null
+++ b/src/feature/hs/hs_client.h
@@ -0,0 +1,92 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_client.h
+ * \brief Header file containing client data for the HS subsytem.
+ **/
+
+#ifndef TOR_HS_CLIENT_H
+#define TOR_HS_CLIENT_H
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "or/hs_descriptor.h"
+#include "or/hs_ident.h"
+
+/* Status code of a descriptor fetch request. */
+typedef enum {
+  /* Something internally went wrong. */
+  HS_CLIENT_FETCH_ERROR        = -1,
+  /* The fetch request has been launched successfully. */
+  HS_CLIENT_FETCH_LAUNCHED     = 0,
+  /* We already have a usable descriptor. No fetch. */
+  HS_CLIENT_FETCH_HAVE_DESC    = 1,
+  /* No more HSDir available to query. */
+  HS_CLIENT_FETCH_NO_HSDIRS    = 2,
+  /* The fetch request is not allowed. */
+  HS_CLIENT_FETCH_NOT_ALLOWED  = 3,
+  /* We are missing information to be able to launch a request. */
+  HS_CLIENT_FETCH_MISSING_INFO = 4,
+  /* There is a pending fetch for the requested service. */
+  HS_CLIENT_FETCH_PENDING      = 5,
+} hs_client_fetch_status_t;
+
+void hs_client_note_connection_attempt_succeeded(
+                                       const edge_connection_t *conn);
+
+int hs_client_decode_descriptor(
+                     const char *desc_str,
+                     const ed25519_public_key_t *service_identity_pk,
+                     hs_descriptor_t **desc);
+int hs_client_any_intro_points_usable(const ed25519_public_key_t *service_pk,
+                                      const hs_descriptor_t *desc);
+int hs_client_refetch_hsdesc(const ed25519_public_key_t *identity_pk);
+void hs_client_dir_info_changed(void);
+
+int hs_client_send_introduce1(origin_circuit_t *intro_circ,
+                              origin_circuit_t *rend_circ);
+
+void hs_client_circuit_has_opened(origin_circuit_t *circ);
+
+int hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
+                                       const uint8_t *payload,
+                                       size_t payload_len);
+int hs_client_receive_introduce_ack(origin_circuit_t *circ,
+                                    const uint8_t *payload,
+                                    size_t payload_len);
+int hs_client_receive_rendezvous2(origin_circuit_t *circ,
+                                  const uint8_t *payload,
+                                  size_t payload_len);
+
+void hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident);
+
+extend_info_t *hs_client_get_random_intro_from_edge(
+                                          const edge_connection_t *edge_conn);
+
+int hs_client_reextend_intro_circuit(origin_circuit_t *circ);
+
+void hs_client_purge_state(void);
+
+void hs_client_free_all(void);
+
+#ifdef HS_CLIENT_PRIVATE
+
+STATIC routerstatus_t *
+pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk);
+
+STATIC extend_info_t *
+client_get_random_intro(const ed25519_public_key_t *service_pk);
+
+STATIC extend_info_t *
+desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip);
+
+STATIC int handle_rendezvous2(origin_circuit_t *circ, const uint8_t *payload,
+                              size_t payload_len);
+
+MOCK_DECL(STATIC hs_client_fetch_status_t,
+          fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk));
+
+#endif /* defined(HS_CLIENT_PRIVATE) */
+
+#endif /* !defined(TOR_HS_CLIENT_H) */
+
diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c
new file mode 100644
index 0000000000..d91f45a639
--- /dev/null
+++ b/src/feature/hs/hs_common.c
@@ -0,0 +1,1825 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_common.c
+ * \brief Contains code shared between different HS protocol version as well
+ *        as useful data structures and accessors used by other subsystems.
+ *        The rendcommon.c should only contains code relating to the v2
+ *        protocol.
+ **/
+
+#define HS_COMMON_PRIVATE
+
+#include "or/or.h"
+
+#include "or/config.h"
+#include "or/circuitbuild.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/hs_cache.h"
+#include "or/hs_common.h"
+#include "or/hs_client.h"
+#include "or/hs_ident.h"
+#include "or/hs_service.h"
+#include "or/hs_circuitmap.h"
+#include "or/policies.h"
+#include "or/rendcommon.h"
+#include "or/rendservice.h"
+#include "or/routerset.h"
+#include "or/router.h"
+#include "or/shared_random_client.h"
+#include "or/dirauth/shared_random_state.h"
+
+#include "or/edge_connection_st.h"
+#include "or/networkstatus_st.h"
+#include "or/node_st.h"
+#include "or/origin_circuit_st.h"
+#include "or/routerstatus_st.h"
+
+/* Trunnel */
+#include "trunnel/ed25519_cert.h"
+
+/* Ed25519 Basepoint value. Taken from section 5 of
+ * https://tools.ietf.org/html/draft-josefsson-eddsa-ed25519-03 */
+static const char *str_ed25519_basepoint =
+  "(15112221349535400772501151409588531511"
+  "454012693041857206046113283949847762202, "
+  "463168356949264781694283940034751631413"
+  "07993866256225615783033603165251855960)";
+
+#ifdef HAVE_SYS_UN_H
+
+/** Given <b>ports</b>, a smarlist containing rend_service_port_config_t,
+ * add the given <b>p</b>, a AF_UNIX port to the list. Return 0 on success
+ * else return -ENOSYS if AF_UNIX is not supported (see function in the
+ * #else statement below). */
+static int
+add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
+{
+  tor_assert(ports);
+  tor_assert(p);
+  tor_assert(p->is_unix_addr);
+
+  smartlist_add(ports, p);
+  return 0;
+}
+
+/** Given <b>conn</b> set it to use the given port <b>p</b> values. Return 0
+ * on success else return -ENOSYS if AF_UNIX is not supported (see function
+ * in the #else statement below). */
+static int
+set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
+{
+  tor_assert(conn);
+  tor_assert(p);
+  tor_assert(p->is_unix_addr);
+
+  conn->base_.socket_family = AF_UNIX;
+  tor_addr_make_unspec(&conn->base_.addr);
+  conn->base_.port = 1;
+  conn->base_.address = tor_strdup(p->unix_addr);
+  return 0;
+}
+
+#else /* !(defined(HAVE_SYS_UN_H)) */
+
+static int
+set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
+{
+  (void) conn;
+  (void) p;
+  return -ENOSYS;
+}
+
+static int
+add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
+{
+  (void) ports;
+  (void) p;
+  return -ENOSYS;
+}
+
+#endif /* defined(HAVE_SYS_UN_H) */
+
+/* Helper function: The key is a digest that we compare to a node_t object
+ * current hsdir_index. */
+static int
+compare_digest_to_fetch_hsdir_index(const void *_key, const void **_member)
+{
+  const char *key = _key;
+  const node_t *node = *_member;
+  return tor_memcmp(key, node->hsdir_index.fetch, DIGEST256_LEN);
+}
+
+/* Helper function: The key is a digest that we compare to a node_t object
+ * next hsdir_index. */
+static int
+compare_digest_to_store_first_hsdir_index(const void *_key,
+                                          const void **_member)
+{
+  const char *key = _key;
+  const node_t *node = *_member;
+  return tor_memcmp(key, node->hsdir_index.store_first, DIGEST256_LEN);
+}
+
+/* Helper function: The key is a digest that we compare to a node_t object
+ * next hsdir_index. */
+static int
+compare_digest_to_store_second_hsdir_index(const void *_key,
+                                          const void **_member)
+{
+  const char *key = _key;
+  const node_t *node = *_member;
+  return tor_memcmp(key, node->hsdir_index.store_second, DIGEST256_LEN);
+}
+
+/* Helper function: Compare two node_t objects current hsdir_index. */
+static int
+compare_node_fetch_hsdir_index(const void **a, const void **b)
+{
+  const node_t *node1= *a;
+  const node_t *node2 = *b;
+  return tor_memcmp(node1->hsdir_index.fetch,
+                    node2->hsdir_index.fetch,
+                    DIGEST256_LEN);
+}
+
+/* Helper function: Compare two node_t objects next hsdir_index. */
+static int
+compare_node_store_first_hsdir_index(const void **a, const void **b)
+{
+  const node_t *node1= *a;
+  const node_t *node2 = *b;
+  return tor_memcmp(node1->hsdir_index.store_first,
+                    node2->hsdir_index.store_first,
+                    DIGEST256_LEN);
+}
+
+/* Helper function: Compare two node_t objects next hsdir_index. */
+static int
+compare_node_store_second_hsdir_index(const void **a, const void **b)
+{
+  const node_t *node1= *a;
+  const node_t *node2 = *b;
+  return tor_memcmp(node1->hsdir_index.store_second,
+                    node2->hsdir_index.store_second,
+                    DIGEST256_LEN);
+}
+
+/* Allocate and return a string containing the path to filename in directory.
+ * This function will never return NULL. The caller must free this path. */
+char *
+hs_path_from_filename(const char *directory, const char *filename)
+{
+  char *file_path = NULL;
+
+  tor_assert(directory);
+  tor_assert(filename);
+
+  tor_asprintf(&file_path, "%s%s%s", directory, PATH_SEPARATOR, filename);
+  return file_path;
+}
+
+/* Make sure that the directory for <b>service</b> is private, using the config
+ * <b>username</b>.
+ * If <b>create</b> is true:
+ *  - if the directory exists, change permissions if needed,
+ *  - if the directory does not exist, create it with the correct permissions.
+ * If <b>create</b> is false:
+ *  - if the directory exists, check permissions,
+ *  - if the directory does not exist, check if we think we can create it.
+ * Return 0 on success, -1 on failure. */
+int
+hs_check_service_private_dir(const char *username, const char *path,
+                             unsigned int dir_group_readable,
+                             unsigned int create)
+{
+  cpd_check_t check_opts = CPD_NONE;
+
+  tor_assert(path);
+
+  if (create) {
+    check_opts |= CPD_CREATE;
+  } else {
+    check_opts |= CPD_CHECK_MODE_ONLY;
+    check_opts |= CPD_CHECK;
+  }
+  if (dir_group_readable) {
+    check_opts |= CPD_GROUP_READ;
+  }
+  /* Check/create directory */
+  if (check_private_dir(path, check_opts, username) < 0) {
+    return -1;
+  }
+  return 0;
+}
+
+/* Default, minimum, and maximum values for the maximum rendezvous failures
+ * consensus parameter. */
+#define MAX_REND_FAILURES_DEFAULT 2
+#define MAX_REND_FAILURES_MIN 1
+#define MAX_REND_FAILURES_MAX 10
+
+/** How many times will a hidden service operator attempt to connect to
+ * a requested rendezvous point before giving up? */
+int
+hs_get_service_max_rend_failures(void)
+{
+  return networkstatus_get_param(NULL, "hs_service_max_rdv_failures",
+                                 MAX_REND_FAILURES_DEFAULT,
+                                 MAX_REND_FAILURES_MIN,
+                                 MAX_REND_FAILURES_MAX);
+}
+
+/** Get the default HS time period length in minutes from the consensus. */
+STATIC uint64_t
+get_time_period_length(void)
+{
+  /* If we are on a test network, make the time period smaller than normal so
+     that we actually see it rotate. Specifically, make it the same length as
+     an SRV protocol run. */
+  if (get_options()->TestingTorNetwork) {
+    unsigned run_duration = sr_state_get_protocol_run_duration();
+    /* An SRV run should take more than a minute (it's 24 rounds) */
+    tor_assert_nonfatal(run_duration > 60);
+    /* Turn it from seconds to minutes before returning: */
+    return sr_state_get_protocol_run_duration() / 60;
+  }
+
+  int32_t time_period_length = networkstatus_get_param(NULL, "hsdir_interval",
+                                             HS_TIME_PERIOD_LENGTH_DEFAULT,
+                                             HS_TIME_PERIOD_LENGTH_MIN,
+                                             HS_TIME_PERIOD_LENGTH_MAX);
+  /* Make sure it's a positive value. */
+  tor_assert(time_period_length >= 0);
+  /* uint64_t will always be able to contain a int32_t */
+  return (uint64_t) time_period_length;
+}
+
+/** Get the HS time period number at time <b>now</b>. If <b>now</b> is not set,
+ *  we try to get the time ourselves from a live consensus. */
+uint64_t
+hs_get_time_period_num(time_t now)
+{
+  uint64_t time_period_num;
+  time_t current_time;
+
+  /* If no time is specified, set current time based on consensus time, and
+   * only fall back to system time if that fails. */
+  if (now != 0) {
+    current_time = now;
+  } else {
+    networkstatus_t *ns = networkstatus_get_live_consensus(approx_time());
+    current_time = ns ? ns->valid_after : approx_time();
+  }
+
+  /* Start by calculating minutes since the epoch */
+  uint64_t time_period_length = get_time_period_length();
+  uint64_t minutes_since_epoch = current_time / 60;
+
+  /* Apply the rotation offset as specified by prop224 (section
+   * [TIME-PERIODS]), so that new time periods synchronize nicely with SRV
+   * publication */
+  unsigned int time_period_rotation_offset = sr_state_get_phase_duration();
+  time_period_rotation_offset /= 60; /* go from seconds to minutes */
+  tor_assert(minutes_since_epoch > time_period_rotation_offset);
+  minutes_since_epoch -= time_period_rotation_offset;
+
+  /* Calculate the time period */
+  time_period_num = minutes_since_epoch / time_period_length;
+  return time_period_num;
+}
+
+/** Get the number of the _upcoming_ HS time period, given that the current
+ *  time is <b>now</b>. If <b>now</b> is not set, we try to get the time from a
+ *  live consensus. */
+uint64_t
+hs_get_next_time_period_num(time_t now)
+{
+  return hs_get_time_period_num(now) + 1;
+}
+
+/* Get the number of the _previous_ HS time period, given that the current time
+ * is <b>now</b>. If <b>now</b> is not set, we try to get the time from a live
+ * consensus. */
+uint64_t
+hs_get_previous_time_period_num(time_t now)
+{
+  return hs_get_time_period_num(now) - 1;
+}
+
+/* Return the start time of the upcoming time period based on <b>now</b>. If
+   <b>now</b> is not set, we try to get the time ourselves from a live
+   consensus. */
+time_t
+hs_get_start_time_of_next_time_period(time_t now)
+{
+  uint64_t time_period_length = get_time_period_length();
+
+  /* Get start time of next time period */
+  uint64_t next_time_period_num = hs_get_next_time_period_num(now);
+  uint64_t start_of_next_tp_in_mins = next_time_period_num *time_period_length;
+
+  /* Apply rotation offset as specified by prop224 section [TIME-PERIODS] */
+  unsigned int time_period_rotation_offset = sr_state_get_phase_duration();
+  return (time_t)(start_of_next_tp_in_mins * 60 + time_period_rotation_offset);
+}
+
+/* Create a new rend_data_t for a specific given <b>version</b>.
+ * Return a pointer to the newly allocated data structure. */
+static rend_data_t *
+rend_data_alloc(uint32_t version)
+{
+  rend_data_t *rend_data = NULL;
+
+  switch (version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2 = tor_malloc_zero(sizeof(*v2));
+    v2->base_.version = HS_VERSION_TWO;
+    v2->base_.hsdirs_fp = smartlist_new();
+    rend_data = &v2->base_;
+    break;
+  }
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  return rend_data;
+}
+
+/** Free all storage associated with <b>data</b> */
+void
+rend_data_free_(rend_data_t *data)
+{
+  if (!data) {
+    return;
+  }
+  /* By using our allocation function, this should always be set. */
+  tor_assert(data->hsdirs_fp);
+  /* Cleanup the HSDir identity digest. */
+  SMARTLIST_FOREACH(data->hsdirs_fp, char *, d, tor_free(d));
+  smartlist_free(data->hsdirs_fp);
+  /* Depending on the version, cleanup. */
+  switch (data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = TO_REND_DATA_V2(data);
+    tor_free(v2_data);
+    break;
+  }
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Allocate and return a deep copy of <b>data</b>. */
+rend_data_t *
+rend_data_dup(const rend_data_t *data)
+{
+  rend_data_t *data_dup = NULL;
+  smartlist_t *hsdirs_fp = smartlist_new();
+
+  tor_assert(data);
+  tor_assert(data->hsdirs_fp);
+
+  SMARTLIST_FOREACH(data->hsdirs_fp, char *, fp,
+                    smartlist_add(hsdirs_fp, tor_memdup(fp, DIGEST_LEN)));
+
+  switch (data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = tor_memdup(TO_REND_DATA_V2(data),
+                                         sizeof(*v2_data));
+    data_dup = &v2_data->base_;
+    data_dup->hsdirs_fp = hsdirs_fp;
+    break;
+  }
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  return data_dup;
+}
+
+/* Compute the descriptor ID for each HS descriptor replica and save them. A
+ * valid onion address must be present in the <b>rend_data</b>.
+ *
+ * Return 0 on success else -1. */
+static int
+compute_desc_id(rend_data_t *rend_data)
+{
+  int ret = 0;
+  unsigned replica;
+  time_t now = time(NULL);
+
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = TO_REND_DATA_V2(rend_data);
+    /* Compute descriptor ID for each replicas. */
+    for (replica = 0; replica < ARRAY_LENGTH(v2_data->descriptor_id);
+         replica++) {
+      ret = rend_compute_v2_desc_id(v2_data->descriptor_id[replica],
+                                    v2_data->onion_address,
+                                    v2_data->descriptor_cookie,
+                                    now, replica);
+      if (ret < 0) {
+        goto end;
+      }
+    }
+    break;
+  }
+  default:
+    tor_assert(0);
+  }
+
+ end:
+  return ret;
+}
+
+/* Allocate and initialize a rend_data_t object for a service using the
+ * provided arguments. All arguments are optional (can be NULL), except from
+ * <b>onion_address</b> which MUST be set. The <b>pk_digest</b> is the hash of
+ * the service private key. The <b>cookie</b> is the rendezvous cookie and
+ * <b>auth_type</b> is which authentiation this service is configured with.
+ *
+ * Return a valid rend_data_t pointer. This only returns a version 2 object of
+ * rend_data_t. */
+rend_data_t *
+rend_data_service_create(const char *onion_address, const char *pk_digest,
+                         const uint8_t *cookie, rend_auth_type_t auth_type)
+{
+  /* Create a rend_data_t object for version 2. */
+  rend_data_t *rend_data = rend_data_alloc(HS_VERSION_TWO);
+  rend_data_v2_t *v2= TO_REND_DATA_V2(rend_data);
+
+  /* We need at least one else the call is wrong. */
+  tor_assert(onion_address != NULL);
+
+  if (pk_digest) {
+    memcpy(v2->rend_pk_digest, pk_digest, sizeof(v2->rend_pk_digest));
+  }
+  if (cookie) {
+    memcpy(rend_data->rend_cookie, cookie, sizeof(rend_data->rend_cookie));
+  }
+
+  strlcpy(v2->onion_address, onion_address, sizeof(v2->onion_address));
+  v2->auth_type = auth_type;
+
+  return rend_data;
+}
+
+/* Allocate and initialize a rend_data_t object for a client request using the
+ * given arguments. Either an onion address or a descriptor ID is needed. Both
+ * can be given but in this case only the onion address will be used to make
+ * the descriptor fetch. The <b>cookie</b> is the rendezvous cookie and
+ * <b>auth_type</b> is which authentiation the service is configured with.
+ *
+ * Return a valid rend_data_t pointer or NULL on error meaning the
+ * descriptor IDs couldn't be computed from the given data. */
+rend_data_t *
+rend_data_client_create(const char *onion_address, const char *desc_id,
+                        const char *cookie, rend_auth_type_t auth_type)
+{
+  /* Create a rend_data_t object for version 2. */
+  rend_data_t *rend_data = rend_data_alloc(HS_VERSION_TWO);
+  rend_data_v2_t *v2= TO_REND_DATA_V2(rend_data);
+
+  /* We need at least one else the call is wrong. */
+  tor_assert(onion_address != NULL || desc_id != NULL);
+
+  if (cookie) {
+    memcpy(v2->descriptor_cookie, cookie, sizeof(v2->descriptor_cookie));
+  }
+  if (desc_id) {
+    memcpy(v2->desc_id_fetch, desc_id, sizeof(v2->desc_id_fetch));
+  }
+  if (onion_address) {
+    strlcpy(v2->onion_address, onion_address, sizeof(v2->onion_address));
+    if (compute_desc_id(rend_data) < 0) {
+      goto error;
+    }
+  }
+
+  v2->auth_type = auth_type;
+
+  return rend_data;
+
+ error:
+  rend_data_free(rend_data);
+  return NULL;
+}
+
+/* Return the onion address from the rend data. Depending on the version,
+ * the size of the address can vary but it's always NUL terminated. */
+const char *
+rend_data_get_address(const rend_data_t *rend_data)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+    return TO_REND_DATA_V2(rend_data)->onion_address;
+  default:
+    /* We should always have a supported version. */
+    tor_assert(0);
+  }
+}
+
+/* Return the descriptor ID for a specific replica number from the rend
+ * data. The returned data is a binary digest and depending on the version its
+ * size can vary. The size of the descriptor ID is put in <b>len_out</b> if
+ * non NULL. */
+const char *
+rend_data_get_desc_id(const rend_data_t *rend_data, uint8_t replica,
+                      size_t *len_out)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+    tor_assert(replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS);
+    if (len_out) {
+      *len_out = DIGEST_LEN;
+    }
+    return TO_REND_DATA_V2(rend_data)->descriptor_id[replica];
+  default:
+    /* We should always have a supported version. */
+    tor_assert(0);
+  }
+}
+
+/* Return the public key digest using the given <b>rend_data</b>. The size of
+ * the digest is put in <b>len_out</b> (if set) which can differ depending on
+ * the version. */
+const uint8_t *
+rend_data_get_pk_digest(const rend_data_t *rend_data, size_t *len_out)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+  {
+    const rend_data_v2_t *v2_data = TO_REND_DATA_V2(rend_data);
+    if (len_out) {
+      *len_out = sizeof(v2_data->rend_pk_digest);
+    }
+    return (const uint8_t *) v2_data->rend_pk_digest;
+  }
+  default:
+    /* We should always have a supported version. */
+    tor_assert(0);
+  }
+}
+
+/* Using the given time period number, compute the disaster shared random
+ * value and put it in srv_out. It MUST be at least DIGEST256_LEN bytes. */
+static void
+compute_disaster_srv(uint64_t time_period_num, uint8_t *srv_out)
+{
+  crypto_digest_t *digest;
+
+  tor_assert(srv_out);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+
+  /* Start setting up payload:
+   *  H("shared-random-disaster" | INT_8(period_length) | INT_8(period_num)) */
+  crypto_digest_add_bytes(digest, HS_SRV_DISASTER_PREFIX,
+                          HS_SRV_DISASTER_PREFIX_LEN);
+
+  /* Setup INT_8(period_length) | INT_8(period_num) */
+  {
+    uint64_t time_period_length = get_time_period_length();
+    char period_stuff[sizeof(uint64_t)*2];
+    size_t offset = 0;
+    set_uint64(period_stuff, tor_htonll(time_period_length));
+    offset += sizeof(uint64_t);
+    set_uint64(period_stuff+offset, tor_htonll(time_period_num));
+    offset += sizeof(uint64_t);
+    tor_assert(offset == sizeof(period_stuff));
+
+    crypto_digest_add_bytes(digest, period_stuff,  sizeof(period_stuff));
+  }
+
+  crypto_digest_get_digest(digest, (char *) srv_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+}
+
+/** Due to the high cost of computing the disaster SRV and that potentially we
+ *  would have to do it thousands of times in a row, we always cache the
+ *  computer disaster SRV (and its corresponding time period num) in case we
+ *  want to reuse it soon after. We need to cache two SRVs, one for each active
+ *  time period.
+ */
+static uint8_t cached_disaster_srv[2][DIGEST256_LEN];
+static uint64_t cached_time_period_nums[2] = {0};
+
+/** Compute the disaster SRV value for this <b>time_period_num</b> and put it
+ *  in <b>srv_out</b> (of size at least DIGEST256_LEN). First check our caches
+ *  to see if we have already computed it. */
+STATIC void
+get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out)
+{
+  if (time_period_num == cached_time_period_nums[0]) {
+    memcpy(srv_out, cached_disaster_srv[0], DIGEST256_LEN);
+    return;
+  } else if (time_period_num == cached_time_period_nums[1]) {
+    memcpy(srv_out, cached_disaster_srv[1], DIGEST256_LEN);
+    return;
+  } else {
+    int replace_idx;
+    // Replace the lower period number.
+    if (cached_time_period_nums[0] <= cached_time_period_nums[1]) {
+      replace_idx = 0;
+    } else {
+      replace_idx = 1;
+    }
+    cached_time_period_nums[replace_idx] = time_period_num;
+    compute_disaster_srv(time_period_num, cached_disaster_srv[replace_idx]);
+    memcpy(srv_out, cached_disaster_srv[replace_idx], DIGEST256_LEN);
+    return;
+  }
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/** Get the first cached disaster SRV. Only used by unittests. */
+STATIC uint8_t *
+get_first_cached_disaster_srv(void)
+{
+  return cached_disaster_srv[0];
+}
+
+/** Get the second cached disaster SRV. Only used by unittests. */
+STATIC uint8_t *
+get_second_cached_disaster_srv(void)
+{
+  return cached_disaster_srv[1];
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/* When creating a blinded key, we need a parameter which construction is as
+ * follow: H(pubkey | [secret] | ed25519-basepoint | nonce).
+ *
+ * The nonce has a pre-defined format which uses the time period number
+ * period_num and the start of the period in second start_time_period.
+ *
+ * The secret of size secret_len is optional meaning that it can be NULL and
+ * thus will be ignored for the param construction.
+ *
+ * The result is put in param_out. */
+static void
+build_blinded_key_param(const ed25519_public_key_t *pubkey,
+                        const uint8_t *secret, size_t secret_len,
+                        uint64_t period_num, uint64_t period_length,
+                        uint8_t *param_out)
+{
+  size_t offset = 0;
+  const char blind_str[] = "Derive temporary signing key";
+  uint8_t nonce[HS_KEYBLIND_NONCE_LEN];
+  crypto_digest_t *digest;
+
+  tor_assert(pubkey);
+  tor_assert(param_out);
+
+  /* Create the nonce N. The construction is as follow:
+   *    N = "key-blind" || INT_8(period_num) || INT_8(period_length) */
+  memcpy(nonce, HS_KEYBLIND_NONCE_PREFIX, HS_KEYBLIND_NONCE_PREFIX_LEN);
+  offset += HS_KEYBLIND_NONCE_PREFIX_LEN;
+  set_uint64(nonce + offset, tor_htonll(period_num));
+  offset += sizeof(uint64_t);
+  set_uint64(nonce + offset, tor_htonll(period_length));
+  offset += sizeof(uint64_t);
+  tor_assert(offset == HS_KEYBLIND_NONCE_LEN);
+
+  /* Generate the parameter h and the construction is as follow:
+   *    h = H(BLIND_STRING | pubkey | [secret] | ed25519-basepoint | N) */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, blind_str, sizeof(blind_str));
+  crypto_digest_add_bytes(digest, (char *) pubkey, ED25519_PUBKEY_LEN);
+  /* Optional secret. */
+  if (secret) {
+    crypto_digest_add_bytes(digest, (char *) secret, secret_len);
+  }
+  crypto_digest_add_bytes(digest, str_ed25519_basepoint,
+                          strlen(str_ed25519_basepoint));
+  crypto_digest_add_bytes(digest, (char *) nonce, sizeof(nonce));
+
+  /* Extract digest and put it in the param. */
+  crypto_digest_get_digest(digest, (char *) param_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+
+  memwipe(nonce, 0, sizeof(nonce));
+}
+
+/* Using an ed25519 public key and version to build the checksum of an
+ * address. Put in checksum_out. Format is:
+ *    SHA3-256(".onion checksum" || PUBKEY || VERSION)
+ *
+ * checksum_out must be large enough to receive 32 bytes (DIGEST256_LEN). */
+static void
+build_hs_checksum(const ed25519_public_key_t *key, uint8_t version,
+                  uint8_t *checksum_out)
+{
+  size_t offset = 0;
+  char data[HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN];
+
+  /* Build checksum data. */
+  memcpy(data, HS_SERVICE_ADDR_CHECKSUM_PREFIX,
+         HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN);
+  offset += HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN;
+  memcpy(data + offset, key->pubkey, ED25519_PUBKEY_LEN);
+  offset += ED25519_PUBKEY_LEN;
+  set_uint8(data + offset, version);
+  offset += sizeof(version);
+  tor_assert(offset == HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN);
+
+  /* Hash the data payload to create the checksum. */
+  crypto_digest256((char *) checksum_out, data, sizeof(data),
+                   DIGEST_SHA3_256);
+}
+
+/* Using an ed25519 public key, checksum and version to build the binary
+ * representation of a service address. Put in addr_out. Format is:
+ *    addr_out = PUBKEY || CHECKSUM || VERSION
+ *
+ * addr_out must be large enough to receive HS_SERVICE_ADDR_LEN bytes. */
+static void
+build_hs_address(const ed25519_public_key_t *key, const uint8_t *checksum,
+                 uint8_t version, char *addr_out)
+{
+  size_t offset = 0;
+
+  tor_assert(key);
+  tor_assert(checksum);
+
+  memcpy(addr_out, key->pubkey, ED25519_PUBKEY_LEN);
+  offset += ED25519_PUBKEY_LEN;
+  memcpy(addr_out + offset, checksum, HS_SERVICE_ADDR_CHECKSUM_LEN_USED);
+  offset += HS_SERVICE_ADDR_CHECKSUM_LEN_USED;
+  set_uint8(addr_out + offset, version);
+  offset += sizeof(uint8_t);
+  tor_assert(offset == HS_SERVICE_ADDR_LEN);
+}
+
+/* Helper for hs_parse_address(): Using a binary representation of a service
+ * address, parse its content into the key_out, checksum_out and version_out.
+ * Any out variable can be NULL in case the caller would want only one field.
+ * checksum_out MUST at least be 2 bytes long. address must be at least
+ * HS_SERVICE_ADDR_LEN bytes but doesn't need to be NUL terminated. */
+static void
+hs_parse_address_impl(const char *address, ed25519_public_key_t *key_out,
+                      uint8_t *checksum_out, uint8_t *version_out)
+{
+  size_t offset = 0;
+
+  tor_assert(address);
+
+  if (key_out) {
+    /* First is the key. */
+    memcpy(key_out->pubkey, address, ED25519_PUBKEY_LEN);
+  }
+  offset += ED25519_PUBKEY_LEN;
+  if (checksum_out) {
+    /* Followed by a 2 bytes checksum. */
+    memcpy(checksum_out, address + offset, HS_SERVICE_ADDR_CHECKSUM_LEN_USED);
+  }
+  offset += HS_SERVICE_ADDR_CHECKSUM_LEN_USED;
+  if (version_out) {
+    /* Finally, version value is 1 byte. */
+    *version_out = get_uint8(address + offset);
+  }
+  offset += sizeof(uint8_t);
+  /* Extra safety. */
+  tor_assert(offset == HS_SERVICE_ADDR_LEN);
+}
+
+/* Using the given identity public key and a blinded public key, compute the
+ * subcredential and put it in subcred_out (must be of size DIGEST256_LEN).
+ * This can't fail. */
+void
+hs_get_subcredential(const ed25519_public_key_t *identity_pk,
+                     const ed25519_public_key_t *blinded_pk,
+                     uint8_t *subcred_out)
+{
+  uint8_t credential[DIGEST256_LEN];
+  crypto_digest_t *digest;
+
+  tor_assert(identity_pk);
+  tor_assert(blinded_pk);
+  tor_assert(subcred_out);
+
+  /* First, build the credential. Construction is as follow:
+   *  credential = H("credential" | public-identity-key) */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HS_CREDENTIAL_PREFIX,
+                          HS_CREDENTIAL_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) identity_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+  crypto_digest_get_digest(digest, (char *) credential, DIGEST256_LEN);
+  crypto_digest_free(digest);
+
+  /* Now, compute the subcredential. Construction is as follow:
+   *  subcredential = H("subcredential" | credential | blinded-public-key). */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HS_SUBCREDENTIAL_PREFIX,
+                          HS_SUBCREDENTIAL_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) credential,
+                          sizeof(credential));
+  crypto_digest_add_bytes(digest, (const char *) blinded_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+  crypto_digest_get_digest(digest, (char *) subcred_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+
+  memwipe(credential, 0, sizeof(credential));
+}
+
+/* From the given list of hidden service ports, find the ones that much the
+ * given edge connection conn, pick one at random and use it to set the
+ * connection address. Return 0 on success or -1 if none. */
+int
+hs_set_conn_addr_port(const smartlist_t *ports, edge_connection_t *conn)
+{
+  rend_service_port_config_t *chosen_port;
+  unsigned int warn_once = 0;
+  smartlist_t *matching_ports;
+
+  tor_assert(ports);
+  tor_assert(conn);
+
+  matching_ports = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(ports, rend_service_port_config_t *, p) {
+    if (TO_CONN(conn)->port != p->virtual_port) {
+      continue;
+    }
+    if (!(p->is_unix_addr)) {
+      smartlist_add(matching_ports, p);
+    } else {
+      if (add_unix_port(matching_ports, p)) {
+        if (!warn_once) {
+          /* Unix port not supported so warn only once. */
+          log_warn(LD_REND, "Saw AF_UNIX virtual port mapping for port %d "
+                            "which is unsupported on this platform. "
+                            "Ignoring it.",
+                   TO_CONN(conn)->port);
+        }
+        warn_once++;
+      }
+    }
+  } SMARTLIST_FOREACH_END(p);
+
+  chosen_port = smartlist_choose(matching_ports);
+  smartlist_free(matching_ports);
+  if (chosen_port) {
+    if (!(chosen_port->is_unix_addr)) {
+      /* Get a non-AF_UNIX connection ready for connection_exit_connect() */
+      tor_addr_copy(&TO_CONN(conn)->addr, &chosen_port->real_addr);
+      TO_CONN(conn)->port = chosen_port->real_port;
+    } else {
+      if (set_unix_port(conn, chosen_port)) {
+        /* Simply impossible to end up here else we were able to add a Unix
+         * port without AF_UNIX support... ? */
+        tor_assert(0);
+      }
+    }
+  }
+  return (chosen_port) ? 0 : -1;
+}
+
+/* Using a base32 representation of a service address, parse its content into
+ * the key_out, checksum_out and version_out. Any out variable can be NULL in
+ * case the caller would want only one field. checksum_out MUST at least be 2
+ * bytes long.
+ *
+ * Return 0 if parsing went well; return -1 in case of error. */
+int
+hs_parse_address(const char *address, ed25519_public_key_t *key_out,
+                 uint8_t *checksum_out, uint8_t *version_out)
+{
+  char decoded[HS_SERVICE_ADDR_LEN];
+
+  tor_assert(address);
+
+  /* Obvious length check. */
+  if (strlen(address) != HS_SERVICE_ADDR_LEN_BASE32) {
+    log_warn(LD_REND, "Service address %s has an invalid length. "
+                      "Expected %lu but got %lu.",
+             escaped_safe_str(address),
+             (unsigned long) HS_SERVICE_ADDR_LEN_BASE32,
+             (unsigned long) strlen(address));
+    goto invalid;
+  }
+
+  /* Decode address so we can extract needed fields. */
+  if (base32_decode(decoded, sizeof(decoded), address, strlen(address)) < 0) {
+    log_warn(LD_REND, "Service address %s can't be decoded.",
+             escaped_safe_str(address));
+    goto invalid;
+  }
+
+  /* Parse the decoded address into the fields we need. */
+  hs_parse_address_impl(decoded, key_out, checksum_out, version_out);
+
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* Validate a given onion address. The length, the base32 decoding and
+ * checksum are validated. Return 1 if valid else 0. */
+int
+hs_address_is_valid(const char *address)
+{
+  uint8_t version;
+  uint8_t checksum[HS_SERVICE_ADDR_CHECKSUM_LEN_USED];
+  uint8_t target_checksum[DIGEST256_LEN];
+  ed25519_public_key_t service_pubkey;
+
+  /* Parse the decoded address into the fields we need. */
+  if (hs_parse_address(address, &service_pubkey, checksum, &version) < 0) {
+    goto invalid;
+  }
+
+  /* Get the checksum it's suppose to be and compare it with what we have
+   * encoded in the address. */
+  build_hs_checksum(&service_pubkey, version, target_checksum);
+  if (tor_memcmp(checksum, target_checksum, sizeof(checksum))) {
+    log_warn(LD_REND, "Service address %s invalid checksum.",
+             escaped_safe_str(address));
+    goto invalid;
+  }
+
+  /* Validate that this pubkey does not have a torsion component. We need to do
+   * this on the prop224 client-side so that attackers can't give equivalent
+   * forms of an onion address to users. */
+  if (ed25519_validate_pubkey(&service_pubkey) < 0) {
+    log_warn(LD_REND, "Service address %s has bad pubkey .",
+             escaped_safe_str(address));
+    goto invalid;
+  }
+
+  /* Valid address. */
+  return 1;
+ invalid:
+  return 0;
+}
+
+/* Build a service address using an ed25519 public key and a given version.
+ * The returned address is base32 encoded and put in addr_out. The caller MUST
+ * make sure the addr_out is at least HS_SERVICE_ADDR_LEN_BASE32 + 1 long.
+ *
+ * Format is as follow:
+ *     base32(PUBKEY || CHECKSUM || VERSION)
+ *     CHECKSUM = H(".onion checksum" || PUBKEY || VERSION)
+ * */
+void
+hs_build_address(const ed25519_public_key_t *key, uint8_t version,
+                 char *addr_out)
+{
+  uint8_t checksum[DIGEST256_LEN];
+  char address[HS_SERVICE_ADDR_LEN];
+
+  tor_assert(key);
+  tor_assert(addr_out);
+
+  /* Get the checksum of the address. */
+  build_hs_checksum(key, version, checksum);
+  /* Get the binary address representation. */
+  build_hs_address(key, checksum, version, address);
+
+  /* Encode the address. addr_out will be NUL terminated after this. */
+  base32_encode(addr_out, HS_SERVICE_ADDR_LEN_BASE32 + 1, address,
+                sizeof(address));
+  /* Validate what we just built. */
+  tor_assert(hs_address_is_valid(addr_out));
+}
+
+/* Return a newly allocated copy of lspec. */
+link_specifier_t *
+hs_link_specifier_dup(const link_specifier_t *lspec)
+{
+  link_specifier_t *result = link_specifier_new();
+  memcpy(result, lspec, sizeof(*result));
+  /* The unrecognized field is a dynamic array so make sure to copy its
+   * content and not the pointer. */
+  link_specifier_setlen_un_unrecognized(
+                  result, link_specifier_getlen_un_unrecognized(lspec));
+  if (link_specifier_getlen_un_unrecognized(result)) {
+    memcpy(link_specifier_getarray_un_unrecognized(result),
+           link_specifier_getconstarray_un_unrecognized(lspec),
+           link_specifier_getlen_un_unrecognized(result));
+  }
+  return result;
+}
+
+/* From a given ed25519 public key pk and an optional secret, compute a
+ * blinded public key and put it in blinded_pk_out. This is only useful to
+ * the client side because the client only has access to the identity public
+ * key of the service. */
+void
+hs_build_blinded_pubkey(const ed25519_public_key_t *pk,
+                        const uint8_t *secret, size_t secret_len,
+                        uint64_t time_period_num,
+                        ed25519_public_key_t *blinded_pk_out)
+{
+  /* Our blinding key API requires a 32 bytes parameter. */
+  uint8_t param[DIGEST256_LEN];
+
+  tor_assert(pk);
+  tor_assert(blinded_pk_out);
+  tor_assert(!tor_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
+
+  build_blinded_key_param(pk, secret, secret_len,
+                          time_period_num, get_time_period_length(), param);
+  ed25519_public_blind(blinded_pk_out, pk, param);
+
+  memwipe(param, 0, sizeof(param));
+}
+
+/* From a given ed25519 keypair kp and an optional secret, compute a blinded
+ * keypair for the current time period and put it in blinded_kp_out. This is
+ * only useful by the service side because the client doesn't have access to
+ * the identity secret key. */
+void
+hs_build_blinded_keypair(const ed25519_keypair_t *kp,
+                         const uint8_t *secret, size_t secret_len,
+                         uint64_t time_period_num,
+                         ed25519_keypair_t *blinded_kp_out)
+{
+  /* Our blinding key API requires a 32 bytes parameter. */
+  uint8_t param[DIGEST256_LEN];
+
+  tor_assert(kp);
+  tor_assert(blinded_kp_out);
+  /* Extra safety. A zeroed key is bad. */
+  tor_assert(!tor_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
+  tor_assert(!tor_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
+
+  build_blinded_key_param(&kp->pubkey, secret, secret_len,
+                          time_period_num, get_time_period_length(), param);
+  ed25519_keypair_blind(blinded_kp_out, kp, param);
+
+  memwipe(param, 0, sizeof(param));
+}
+
+/* Return true if we are currently in the time segment between a new time
+ * period and a new SRV (in the real network that happens between 12:00 and
+ * 00:00 UTC). Here is a diagram showing exactly when this returns true:
+ *
+ *    +------------------------------------------------------------------+
+ *    |                                                                  |
+ *    | 00:00      12:00       00:00       12:00       00:00       12:00 |
+ *    | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+ *    |                                                                  |
+ *    |  $==========|-----------$===========|-----------$===========|    |
+ *    |             ^^^^^^^^^^^^            ^^^^^^^^^^^^                 |
+ *    |                                                                  |
+ *    +------------------------------------------------------------------+
+ */
+MOCK_IMPL(int,
+hs_in_period_between_tp_and_srv,(const networkstatus_t *consensus, time_t now))
+{
+  time_t valid_after;
+  time_t srv_start_time, tp_start_time;
+
+  if (!consensus) {
+    consensus = networkstatus_get_live_consensus(now);
+    if (!consensus) {
+      return 0;
+    }
+  }
+
+  /* Get start time of next TP and of current SRV protocol run, and check if we
+   * are between them. */
+  valid_after = consensus->valid_after;
+  srv_start_time =
+    sr_state_get_start_time_of_current_protocol_run(valid_after);
+  tp_start_time = hs_get_start_time_of_next_time_period(srv_start_time);
+
+  if (valid_after >= srv_start_time && valid_after < tp_start_time) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/* Return 1 if any virtual port in ports needs a circuit with good uptime.
+ * Else return 0. */
+int
+hs_service_requires_uptime_circ(const smartlist_t *ports)
+{
+  tor_assert(ports);
+
+  SMARTLIST_FOREACH_BEGIN(ports, rend_service_port_config_t *, p) {
+    if (smartlist_contains_int_as_string(get_options()->LongLivedPorts,
+                                         p->virtual_port)) {
+      return 1;
+    }
+  } SMARTLIST_FOREACH_END(p);
+  return 0;
+}
+
+/* Build hs_index which is used to find the responsible hsdirs. This index
+ * value is used to select the responsible HSDir where their hsdir_index is
+ * closest to this value.
+ *    SHA3-256("store-at-idx" | blinded_public_key |
+ *             INT_8(replicanum) | INT_8(period_length) | INT_8(period_num) )
+ *
+ * hs_index_out must be large enough to receive DIGEST256_LEN bytes. */
+void
+hs_build_hs_index(uint64_t replica, const ed25519_public_key_t *blinded_pk,
+                  uint64_t period_num, uint8_t *hs_index_out)
+{
+  crypto_digest_t *digest;
+
+  tor_assert(blinded_pk);
+  tor_assert(hs_index_out);
+
+  /* Build hs_index. See construction at top of function comment. */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HS_INDEX_PREFIX, HS_INDEX_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) blinded_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+
+  /* Now setup INT_8(replicanum) | INT_8(period_length) | INT_8(period_num) */
+  {
+    uint64_t period_length = get_time_period_length();
+    char buf[sizeof(uint64_t)*3];
+    size_t offset = 0;
+    set_uint64(buf, tor_htonll(replica));
+    offset += sizeof(uint64_t);
+    set_uint64(buf+offset, tor_htonll(period_length));
+    offset += sizeof(uint64_t);
+    set_uint64(buf+offset, tor_htonll(period_num));
+    offset += sizeof(uint64_t);
+    tor_assert(offset == sizeof(buf));
+
+    crypto_digest_add_bytes(digest, buf, sizeof(buf));
+  }
+
+  crypto_digest_get_digest(digest, (char *) hs_index_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+}
+
+/* Build hsdir_index which is used to find the responsible hsdirs. This is the
+ * index value that is compare to the hs_index when selecting an HSDir.
+ *    SHA3-256("node-idx" | node_identity |
+ *             shared_random_value | INT_8(period_length) | INT_8(period_num) )
+ *
+ * hsdir_index_out must be large enough to receive DIGEST256_LEN bytes. */
+void
+hs_build_hsdir_index(const ed25519_public_key_t *identity_pk,
+                     const uint8_t *srv_value, uint64_t period_num,
+                     uint8_t *hsdir_index_out)
+{
+  crypto_digest_t *digest;
+
+  tor_assert(identity_pk);
+  tor_assert(srv_value);
+  tor_assert(hsdir_index_out);
+
+  /* Build hsdir_index. See construction at top of function comment. */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HSDIR_INDEX_PREFIX, HSDIR_INDEX_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) identity_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+  crypto_digest_add_bytes(digest, (const char *) srv_value, DIGEST256_LEN);
+
+  {
+    uint64_t time_period_length = get_time_period_length();
+    char period_stuff[sizeof(uint64_t)*2];
+    size_t offset = 0;
+    set_uint64(period_stuff, tor_htonll(period_num));
+    offset += sizeof(uint64_t);
+    set_uint64(period_stuff+offset, tor_htonll(time_period_length));
+    offset += sizeof(uint64_t);
+    tor_assert(offset == sizeof(period_stuff));
+
+    crypto_digest_add_bytes(digest, period_stuff,  sizeof(period_stuff));
+  }
+
+  crypto_digest_get_digest(digest, (char *) hsdir_index_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+}
+
+/* Return a newly allocated buffer containing the current shared random value
+ * or if not present, a disaster value is computed using the given time period
+ * number. If a consensus is provided in <b>ns</b>, use it to get the SRV
+ * value. This function can't fail. */
+uint8_t *
+hs_get_current_srv(uint64_t time_period_num, const networkstatus_t *ns)
+{
+  uint8_t *sr_value = tor_malloc_zero(DIGEST256_LEN);
+  const sr_srv_t *current_srv = sr_get_current(ns);
+
+  if (current_srv) {
+    memcpy(sr_value, current_srv->value, sizeof(current_srv->value));
+  } else {
+    /* Disaster mode. */
+    get_disaster_srv(time_period_num, sr_value);
+  }
+  return sr_value;
+}
+
+/* Return a newly allocated buffer containing the previous shared random
+ * value or if not present, a disaster value is computed using the given time
+ * period number. This function can't fail. */
+uint8_t *
+hs_get_previous_srv(uint64_t time_period_num, const networkstatus_t *ns)
+{
+  uint8_t *sr_value = tor_malloc_zero(DIGEST256_LEN);
+  const sr_srv_t *previous_srv = sr_get_previous(ns);
+
+  if (previous_srv) {
+    memcpy(sr_value, previous_srv->value, sizeof(previous_srv->value));
+  } else {
+    /* Disaster mode. */
+    get_disaster_srv(time_period_num, sr_value);
+  }
+  return sr_value;
+}
+
+/* Return the number of replicas defined by a consensus parameter or the
+ * default value. */
+int32_t
+hs_get_hsdir_n_replicas(void)
+{
+  /* The [1,16] range is a specification requirement. */
+  return networkstatus_get_param(NULL, "hsdir_n_replicas",
+                                 HS_DEFAULT_HSDIR_N_REPLICAS, 1, 16);
+}
+
+/* Return the spread fetch value defined by a consensus parameter or the
+ * default value. */
+int32_t
+hs_get_hsdir_spread_fetch(void)
+{
+  /* The [1,128] range is a specification requirement. */
+  return networkstatus_get_param(NULL, "hsdir_spread_fetch",
+                                 HS_DEFAULT_HSDIR_SPREAD_FETCH, 1, 128);
+}
+
+/* Return the spread store value defined by a consensus parameter or the
+ * default value. */
+int32_t
+hs_get_hsdir_spread_store(void)
+{
+  /* The [1,128] range is a specification requirement. */
+  return networkstatus_get_param(NULL, "hsdir_spread_store",
+                                 HS_DEFAULT_HSDIR_SPREAD_STORE, 1, 128);
+}
+
+/** <b>node</b> is an HSDir so make sure that we have assigned an hsdir index.
+ *  Return 0 if everything is as expected, else return -1. */
+static int
+node_has_hsdir_index(const node_t *node)
+{
+  tor_assert(node_supports_v3_hsdir(node));
+
+  /* A node can't have an HSDir index without a descriptor since we need desc
+   * to get its ed25519 key.  for_direct_connect should be zero, since we
+   * always use the consensus-indexed node's keys to build the hash ring, even
+   * if some of the consensus-indexed nodes are also bridges. */
+  if (!node_has_preferred_descriptor(node, 0)) {
+    return 0;
+  }
+
+  /* At this point, since the node has a desc, this node must also have an
+   * hsdir index. If not, something went wrong, so BUG out. */
+  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch,
+                          DIGEST256_LEN))) {
+    return 0;
+  }
+  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first,
+                          DIGEST256_LEN))) {
+    return 0;
+  }
+  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second,
+                          DIGEST256_LEN))) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/* For a given blinded key and time period number, get the responsible HSDir
+ * and put their routerstatus_t object in the responsible_dirs list. If
+ * 'use_second_hsdir_index' is true, use the second hsdir_index of the node_t
+ * is used. If 'for_fetching' is true, the spread fetch consensus parameter is
+ * used else the spread store is used which is only for upload. This function
+ * can't fail but it is possible that the responsible_dirs list contains fewer
+ * nodes than expected.
+ *
+ * This function goes over the latest consensus routerstatus list and sorts it
+ * by their node_t hsdir_index then does a binary search to find the closest
+ * node. All of this makes it a bit CPU intensive so use it wisely. */
+void
+hs_get_responsible_hsdirs(const ed25519_public_key_t *blinded_pk,
+                          uint64_t time_period_num, int use_second_hsdir_index,
+                          int for_fetching, smartlist_t *responsible_dirs)
+{
+  smartlist_t *sorted_nodes;
+  /* The compare function used for the smartlist bsearch. We have two
+   * different depending on is_next_period. */
+  int (*cmp_fct)(const void *, const void **);
+
+  tor_assert(blinded_pk);
+  tor_assert(responsible_dirs);
+
+  sorted_nodes = smartlist_new();
+
+  /* Make sure we actually have a live consensus */
+  networkstatus_t *c = networkstatus_get_live_consensus(approx_time());
+  if (!c || smartlist_len(c->routerstatus_list) == 0) {
+      log_warn(LD_REND, "No live consensus so we can't get the responsible "
+               "hidden service directories.");
+      goto done;
+  }
+
+  /* Ensure the nodelist is fresh, since it contains the HSDir indices. */
+  nodelist_ensure_freshness(c);
+
+  /* Add every node_t that support HSDir v3 for which we do have a valid
+   * hsdir_index already computed for them for this consensus. */
+  {
+    SMARTLIST_FOREACH_BEGIN(c->routerstatus_list, const routerstatus_t *, rs) {
+      /* Even though this node_t object won't be modified and should be const,
+       * we can't add const object in a smartlist_t. */
+      node_t *n = node_get_mutable_by_id(rs->identity_digest);
+      tor_assert(n);
+      if (node_supports_v3_hsdir(n) && rs->is_hs_dir) {
+        if (!node_has_hsdir_index(n)) {
+          log_info(LD_GENERAL, "Node %s was found without hsdir index.",
+                   node_describe(n));
+          continue;
+        }
+        smartlist_add(sorted_nodes, n);
+      }
+    } SMARTLIST_FOREACH_END(rs);
+  }
+  if (smartlist_len(sorted_nodes) == 0) {
+    log_warn(LD_REND, "No nodes found to be HSDir or supporting v3.");
+    goto done;
+  }
+
+  /* First thing we have to do is sort all node_t by hsdir_index. The
+   * is_next_period tells us if we want the current or the next one. Set the
+   * bsearch compare function also while we are at it. */
+  if (for_fetching) {
+    smartlist_sort(sorted_nodes, compare_node_fetch_hsdir_index);
+    cmp_fct = compare_digest_to_fetch_hsdir_index;
+  } else if (use_second_hsdir_index) {
+    smartlist_sort(sorted_nodes, compare_node_store_second_hsdir_index);
+    cmp_fct = compare_digest_to_store_second_hsdir_index;
+  } else {
+    smartlist_sort(sorted_nodes, compare_node_store_first_hsdir_index);
+    cmp_fct = compare_digest_to_store_first_hsdir_index;
+  }
+
+  /* For all replicas, we'll select a set of HSDirs using the consensus
+   * parameters and the sorted list. The replica starting at value 1 is
+   * defined by the specification. */
+  for (int replica = 1; replica <= hs_get_hsdir_n_replicas(); replica++) {
+    int idx, start, found, n_added = 0;
+    uint8_t hs_index[DIGEST256_LEN] = {0};
+    /* Number of node to add to the responsible dirs list depends on if we are
+     * trying to fetch or store. A client always fetches. */
+    int n_to_add = (for_fetching) ? hs_get_hsdir_spread_fetch() :
+                                    hs_get_hsdir_spread_store();
+
+    /* Get the index that we should use to select the node. */
+    hs_build_hs_index(replica, blinded_pk, time_period_num, hs_index);
+    /* The compare function pointer has been set correctly earlier. */
+    start = idx = smartlist_bsearch_idx(sorted_nodes, hs_index, cmp_fct,
+                                        &found);
+    /* Getting the length of the list if no member is greater than the key we
+     * are looking for so start at the first element. */
+    if (idx == smartlist_len(sorted_nodes)) {
+      start = idx = 0;
+    }
+    while (n_added < n_to_add) {
+      const node_t *node = smartlist_get(sorted_nodes, idx);
+      /* If the node has already been selected which is possible between
+       * replicas, the specification says to skip over. */
+      if (!smartlist_contains(responsible_dirs, node->rs)) {
+        smartlist_add(responsible_dirs, node->rs);
+        ++n_added;
+      }
+      if (++idx == smartlist_len(sorted_nodes)) {
+        /* Wrap if we've reached the end of the list. */
+        idx = 0;
+      }
+      if (idx == start) {
+        /* We've gone over the whole list, stop and avoid infinite loop. */
+        break;
+      }
+    }
+  }
+
+ done:
+  smartlist_free(sorted_nodes);
+}
+
+/*********************** HSDir request tracking ***************************/
+
+/** Return the period for which a hidden service directory cannot be queried
+ * for the same descriptor ID again, taking TestingTorNetwork into account. */
+time_t
+hs_hsdir_requery_period(const or_options_t *options)
+{
+  tor_assert(options);
+
+  if (options->TestingTorNetwork) {
+    return REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING;
+  } else {
+    return REND_HID_SERV_DIR_REQUERY_PERIOD;
+  }
+}
+
+/** Tracks requests for fetching hidden service descriptors. It's used by
+ *  hidden service clients, to avoid querying HSDirs that have already failed
+ *  giving back a descriptor. The same data structure is used to track both v2
+ *  and v3 HS descriptor requests.
+ *
+ * The string map is a key/value store that contains the last request times to
+ * hidden service directories for certain queries. Specifically:
+ *
+ *   key = base32(hsdir_identity) + base32(hs_identity)
+ *   value = time_t of last request for that hs_identity to that HSDir
+ *
+ * where 'hsdir_identity' is the identity digest of the HSDir node, and
+ * 'hs_identity' is the descriptor ID of the HS in the v2 case, or the ed25519
+ * blinded public key of the HS in the v3 case. */
+static strmap_t *last_hid_serv_requests_ = NULL;
+
+/** Returns last_hid_serv_requests_, initializing it to a new strmap if
+ * necessary. */
+STATIC strmap_t *
+get_last_hid_serv_requests(void)
+{
+  if (!last_hid_serv_requests_)
+    last_hid_serv_requests_ = strmap_new();
+  return last_hid_serv_requests_;
+}
+
+/** Look up the last request time to hidden service directory <b>hs_dir</b>
+ * for descriptor request key <b>req_key_str</b> which is the descriptor ID
+ * for a v2 service or the blinded key for v3. If <b>set</b> is non-zero,
+ * assign the current time <b>now</b> and return that.  Otherwise, return the
+ * most recent request time, or 0 if no such request has been sent before. */
+time_t
+hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
+                                const char *req_key_str,
+                                time_t now, int set)
+{
+  char hsdir_id_base32[BASE32_DIGEST_LEN + 1];
+  char *hsdir_desc_comb_id = NULL;
+  time_t *last_request_ptr;
+  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
+
+  /* Create the key */
+  base32_encode(hsdir_id_base32, sizeof(hsdir_id_base32),
+                hs_dir->identity_digest, DIGEST_LEN);
+  tor_asprintf(&hsdir_desc_comb_id, "%s%s", hsdir_id_base32, req_key_str);
+
+  if (set) {
+    time_t *oldptr;
+    last_request_ptr = tor_malloc_zero(sizeof(time_t));
+    *last_request_ptr = now;
+    oldptr = strmap_set(last_hid_serv_requests, hsdir_desc_comb_id,
+                        last_request_ptr);
+    tor_free(oldptr);
+  } else {
+    last_request_ptr = strmap_get(last_hid_serv_requests,
+                                  hsdir_desc_comb_id);
+  }
+
+  tor_free(hsdir_desc_comb_id);
+  return (last_request_ptr) ? *last_request_ptr : 0;
+}
+
+/** Clean the history of request times to hidden service directories, so that
+ * it does not contain requests older than REND_HID_SERV_DIR_REQUERY_PERIOD
+ * seconds any more. */
+void
+hs_clean_last_hid_serv_requests(time_t now)
+{
+  strmap_iter_t *iter;
+  time_t cutoff = now - hs_hsdir_requery_period(get_options());
+  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
+  for (iter = strmap_iter_init(last_hid_serv_requests);
+       !strmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    time_t *ent;
+    strmap_iter_get(iter, &key, &val);
+    ent = (time_t *) val;
+    if (*ent < cutoff) {
+      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
+      tor_free(ent);
+    } else {
+      iter = strmap_iter_next(last_hid_serv_requests, iter);
+    }
+  }
+}
+
+/** Remove all requests related to the descriptor request key string
+ * <b>req_key_str</b> from the history of times of requests to hidden service
+ * directories.
+ *
+ * This is called from rend_client_note_connection_attempt_ended(), which
+ * must be idempotent, so any future changes to this function must leave it
+ * idempotent too. */
+void
+hs_purge_hid_serv_from_last_hid_serv_requests(const char *req_key_str)
+{
+  strmap_iter_t *iter;
+  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
+
+  for (iter = strmap_iter_init(last_hid_serv_requests);
+       !strmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    strmap_iter_get(iter, &key, &val);
+
+    /* XXX: The use of REND_DESC_ID_V2_LEN_BASE32 is very wrong in terms of
+     * semantic, see #23305. */
+
+    /* This strmap contains variable-sized elements so this is a basic length
+     * check on the strings we are about to compare. The key is variable sized
+     * since it's composed as follows:
+     *   key = base32(hsdir_identity) + base32(req_key_str)
+     * where 'req_key_str' is the descriptor ID of the HS in the v2 case, or
+     * the ed25519 blinded public key of the HS in the v3 case. */
+    if (strlen(key) < REND_DESC_ID_V2_LEN_BASE32 + strlen(req_key_str)) {
+      iter = strmap_iter_next(last_hid_serv_requests, iter);
+      continue;
+    }
+
+    /* Check if the tracked request matches our request key */
+    if (tor_memeq(key + REND_DESC_ID_V2_LEN_BASE32, req_key_str,
+                  strlen(req_key_str))) {
+      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
+      tor_free(val);
+    } else {
+      iter = strmap_iter_next(last_hid_serv_requests, iter);
+    }
+  }
+}
+
+/** Purge the history of request times to hidden service directories,
+ * so that future lookups of an HS descriptor will not fail because we
+ * accessed all of the HSDir relays responsible for the descriptor
+ * recently. */
+void
+hs_purge_last_hid_serv_requests(void)
+{
+ /* Don't create the table if it doesn't exist yet (and it may very
+   * well not exist if the user hasn't accessed any HSes)... */
+  strmap_t *old_last_hid_serv_requests = last_hid_serv_requests_;
+  /* ... and let get_last_hid_serv_requests re-create it for us if
+   * necessary. */
+  last_hid_serv_requests_ = NULL;
+
+  if (old_last_hid_serv_requests != NULL) {
+    log_info(LD_REND, "Purging client last-HS-desc-request-time table");
+    strmap_free(old_last_hid_serv_requests, tor_free_);
+  }
+}
+
+/***********************************************************************/
+
+/** Given the list of responsible HSDirs in <b>responsible_dirs</b>, pick the
+ *  one that we should use to fetch a descriptor right now. Take into account
+ *  previous failed attempts at fetching this descriptor from HSDirs using the
+ *  string identifier <b>req_key_str</b>.
+ *
+ *  Steals ownership of <b>responsible_dirs</b>.
+ *
+ *  Return the routerstatus of the chosen HSDir if successful, otherwise return
+ *  NULL if no HSDirs are worth trying right now. */
+routerstatus_t *
+hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
+{
+  smartlist_t *usable_responsible_dirs = smartlist_new();
+  const or_options_t *options = get_options();
+  routerstatus_t *hs_dir;
+  time_t now = time(NULL);
+  int excluded_some;
+
+  tor_assert(req_key_str);
+
+  /* Clean outdated request history first. */
+  hs_clean_last_hid_serv_requests(now);
+
+  /* Only select those hidden service directories to which we did not send a
+   * request recently and for which we have a router descriptor here.
+   *
+   * Use for_direct_connect==0 even if we will be connecting to the node
+   * directly, since we always use the key information in the
+   * consensus-indexed node descriptors for building the index.
+   **/
+  SMARTLIST_FOREACH_BEGIN(responsible_dirs, routerstatus_t *, dir) {
+    time_t last = hs_lookup_last_hid_serv_request(dir, req_key_str, 0, 0);
+    const node_t *node = node_get_by_id(dir->identity_digest);
+    if (last + hs_hsdir_requery_period(options) >= now ||
+        !node || !node_has_preferred_descriptor(node, 0)) {
+      SMARTLIST_DEL_CURRENT(responsible_dirs, dir);
+      continue;
+    }
+    if (!routerset_contains_node(options->ExcludeNodes, node)) {
+      smartlist_add(usable_responsible_dirs, dir);
+    }
+  } SMARTLIST_FOREACH_END(dir);
+
+  excluded_some =
+    smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs);
+
+  hs_dir = smartlist_choose(usable_responsible_dirs);
+  if (!hs_dir && !options->StrictNodes) {
+    hs_dir = smartlist_choose(responsible_dirs);
+  }
+
+  smartlist_free(responsible_dirs);
+  smartlist_free(usable_responsible_dirs);
+  if (!hs_dir) {
+    log_info(LD_REND, "Could not pick one of the responsible hidden "
+                      "service directories, because we requested them all "
+                      "recently without success.");
+    if (options->StrictNodes && excluded_some) {
+      log_warn(LD_REND, "Could not pick a hidden service directory for the "
+               "requested hidden service: they are all either down or "
+               "excluded, and StrictNodes is set.");
+    }
+  } else {
+    /* Remember that we are requesting a descriptor from this hidden service
+     * directory now. */
+    hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1);
+  }
+
+  return hs_dir;
+}
+
+/* From a list of link specifier, an onion key and if we are requesting a
+ * direct connection (ex: single onion service), return a newly allocated
+ * extend_info_t object. This function always returns an extend info with
+ * an IPv4 address, or NULL.
+ *
+ * It performs the following checks:
+ *  if either IPv4 or legacy ID is missing, return NULL.
+ *  if direct_conn, and we can't reach the IPv4 address, return NULL.
+ */
+extend_info_t *
+hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
+                               const curve25519_public_key_t *onion_key,
+                               int direct_conn)
+{
+  int have_v4 = 0, have_legacy_id = 0, have_ed25519_id = 0;
+  char legacy_id[DIGEST_LEN] = {0};
+  uint16_t port_v4 = 0;
+  tor_addr_t addr_v4;
+  ed25519_public_key_t ed25519_pk;
+  extend_info_t *info = NULL;
+
+  tor_assert(lspecs);
+
+  SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
+    switch (link_specifier_get_ls_type(ls)) {
+    case LS_IPV4:
+      /* Skip if we already seen a v4. */
+      if (have_v4) continue;
+      tor_addr_from_ipv4h(&addr_v4,
+                          link_specifier_get_un_ipv4_addr(ls));
+      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      have_v4 = 1;
+      break;
+    case LS_LEGACY_ID:
+      /* Make sure we do have enough bytes for the legacy ID. */
+      if (link_specifier_getlen_un_legacy_id(ls) < sizeof(legacy_id)) {
+        break;
+      }
+      memcpy(legacy_id, link_specifier_getconstarray_un_legacy_id(ls),
+             sizeof(legacy_id));
+      have_legacy_id = 1;
+      break;
+    case LS_ED25519_ID:
+      memcpy(ed25519_pk.pubkey,
+             link_specifier_getconstarray_un_ed25519_id(ls),
+             ED25519_PUBKEY_LEN);
+      have_ed25519_id = 1;
+      break;
+    default:
+      /* Ignore unknown. */
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* Legacy ID is mandatory, and we require IPv4. */
+  if (!have_v4 || !have_legacy_id) {
+    goto done;
+  }
+
+  /* We know we have IPv4, because we just checked. */
+  if (!direct_conn) {
+    /* All clients can extend to any IPv4 via a 3-hop path. */
+    goto validate;
+  } else if (direct_conn &&
+             fascist_firewall_allows_address_addr(&addr_v4, port_v4,
+                                                  FIREWALL_OR_CONNECTION,
+                                                  0, 0)) {
+    /* Direct connection and we can reach it in IPv4 so go for it. */
+    goto validate;
+
+    /* We will add support for falling back to a 3-hop path in a later
+     * release. */
+  } else {
+    /* If we can't reach IPv4, return NULL. */
+    goto done;
+  }
+
+  /* We will add support for IPv6 in a later release. */
+
+ validate:
+  /* We'll validate now that the address we've picked isn't a private one. If
+   * it is, are we allowing to extend to private address? */
+  if (!extend_info_addr_is_allowed(&addr_v4)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Requested address is private and we are not allowed to extend to "
+           "it: %s:%u", fmt_addr(&addr_v4), port_v4);
+    goto done;
+  }
+
+  /* We do have everything for which we think we can connect successfully. */
+  info = extend_info_new(NULL, legacy_id,
+                         (have_ed25519_id) ? &ed25519_pk : NULL, NULL,
+                         onion_key, &addr_v4, port_v4);
+ done:
+  return info;
+}
+
+/***********************************************************************/
+
+/* Initialize the entire HS subsytem. This is called in tor_init() before any
+ * torrc options are loaded. Only for >= v3. */
+void
+hs_init(void)
+{
+  hs_circuitmap_init();
+  hs_service_init();
+  hs_cache_init();
+}
+
+/* Release and cleanup all memory of the HS subsystem (all version). This is
+ * called by tor_free_all(). */
+void
+hs_free_all(void)
+{
+  hs_circuitmap_free_all();
+  hs_service_free_all();
+  hs_cache_free_all();
+  hs_client_free_all();
+}
+
+/* For the given origin circuit circ, decrement the number of rendezvous
+ * stream counter. This handles every hidden service version. */
+void
+hs_dec_rdv_stream_counter(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circ->rend_data) {
+    circ->rend_data->nr_streams--;
+  } else if (circ->hs_ident) {
+    circ->hs_ident->num_rdv_streams--;
+  } else {
+    /* Should not be called if this circuit is not for hidden service. */
+    tor_assert_nonfatal_unreached();
+  }
+}
+
+/* For the given origin circuit circ, increment the number of rendezvous
+ * stream counter. This handles every hidden service version. */
+void
+hs_inc_rdv_stream_counter(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circ->rend_data) {
+    circ->rend_data->nr_streams++;
+  } else if (circ->hs_ident) {
+    circ->hs_ident->num_rdv_streams++;
+  } else {
+    /* Should not be called if this circuit is not for hidden service. */
+    tor_assert_nonfatal_unreached();
+  }
+}
diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h
new file mode 100644
index 0000000000..24f5f3a20f
--- /dev/null
+++ b/src/feature/hs/hs_common.h
@@ -0,0 +1,288 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_common.h
+ * \brief Header file containing common data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_COMMON_H
+#define TOR_HS_COMMON_H
+
+#include "or/or.h"
+#include "lib/defs/x25519_sizes.h"
+
+struct curve25519_public_key_t;
+struct ed25519_public_key_t;
+struct ed25519_keypair_t;
+
+/* Trunnel */
+#include "trunnel/ed25519_cert.h"
+
+/* Protocol version 2. Use this instead of hardcoding "2" in the code base,
+ * this adds a clearer semantic to the value when used. */
+#define HS_VERSION_TWO 2
+/* Version 3 of the protocol (prop224). */
+#define HS_VERSION_THREE 3
+/* Earliest and latest version we support. */
+#define HS_VERSION_MIN HS_VERSION_TWO
+#define HS_VERSION_MAX HS_VERSION_THREE
+
+/** Try to maintain this many intro points per service by default. */
+#define NUM_INTRO_POINTS_DEFAULT 3
+/** Maximum number of intro points per generic and version 2 service. */
+#define NUM_INTRO_POINTS_MAX 10
+/** Number of extra intro points we launch if our set of intro nodes is empty.
+ * See proposal 155, section 4. */
+#define NUM_INTRO_POINTS_EXTRA 2
+
+/** If we can't build our intro circuits, don't retry for this long. */
+#define INTRO_CIRC_RETRY_PERIOD (60*5)
+/** Don't try to build more than this many circuits before giving up for a
+ * while.*/
+#define MAX_INTRO_CIRCS_PER_PERIOD 10
+/** How many times will a hidden service operator attempt to connect to a
+ * requested rendezvous point before giving up? */
+#define MAX_REND_FAILURES 1
+/** How many seconds should we spend trying to connect to a requested
+ * rendezvous point before giving up? */
+#define MAX_REND_TIMEOUT 30
+
+/* String prefix for the signature of ESTABLISH_INTRO */
+#define ESTABLISH_INTRO_SIG_PREFIX "Tor establish-intro cell v1"
+
+/* The default HS time period length */
+#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
+
+/* Prefix of the onion address checksum. */
+#define HS_SERVICE_ADDR_CHECKSUM_PREFIX ".onion checksum"
+/* Length of the checksum prefix minus the NUL terminated byte. */
+#define HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN \
+  (sizeof(HS_SERVICE_ADDR_CHECKSUM_PREFIX) - 1)
+/* Length of the resulting checksum of the address. The construction of this
+ * checksum looks like:
+ *   CHECKSUM = ".onion checksum" || PUBKEY || VERSION
+ * where VERSION is 1 byte. This is pre-hashing. */
+#define HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN \
+  (HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN + ED25519_PUBKEY_LEN + sizeof(uint8_t))
+/* The amount of bytes we use from the address checksum. */
+#define HS_SERVICE_ADDR_CHECKSUM_LEN_USED 2
+/* Length of the binary encoded service address which is of course before the
+ * base32 encoding. Construction is:
+ *    PUBKEY || CHECKSUM || VERSION
+ * with 1 byte VERSION and 2 bytes CHECKSUM. The following is 35 bytes. */
+#define HS_SERVICE_ADDR_LEN \
+  (ED25519_PUBKEY_LEN + HS_SERVICE_ADDR_CHECKSUM_LEN_USED + sizeof(uint8_t))
+/* Length of 'y' portion of 'y.onion' URL. This is base32 encoded and the
+ * length ends up to 56 bytes (not counting the terminated NUL byte.) */
+#define HS_SERVICE_ADDR_LEN_BASE32 \
+  (CEIL_DIV(HS_SERVICE_ADDR_LEN * 8, 5))
+
+/* The default HS time period length */
+#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
+/* The time period rotation offset as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_ROTATION_OFFSET (12 * 60) /* minutes */
+
+/* Keyblinding parameter construction is as follow:
+ *    "key-blind" || INT_8(period_num) || INT_8(start_period_sec) */
+#define HS_KEYBLIND_NONCE_PREFIX "key-blind"
+#define HS_KEYBLIND_NONCE_PREFIX_LEN (sizeof(HS_KEYBLIND_NONCE_PREFIX) - 1)
+#define HS_KEYBLIND_NONCE_LEN \
+  (HS_KEYBLIND_NONCE_PREFIX_LEN + sizeof(uint64_t) + sizeof(uint64_t))
+
+/* Credential and subcredential prefix value. */
+#define HS_CREDENTIAL_PREFIX "credential"
+#define HS_CREDENTIAL_PREFIX_LEN (sizeof(HS_CREDENTIAL_PREFIX) - 1)
+#define HS_SUBCREDENTIAL_PREFIX "subcredential"
+#define HS_SUBCREDENTIAL_PREFIX_LEN (sizeof(HS_SUBCREDENTIAL_PREFIX) - 1)
+
+/* Node hidden service stored at index prefix value. */
+#define HS_INDEX_PREFIX "store-at-idx"
+#define HS_INDEX_PREFIX_LEN (sizeof(HS_INDEX_PREFIX) - 1)
+
+/* Node hidden service directory index prefix value. */
+#define HSDIR_INDEX_PREFIX "node-idx"
+#define HSDIR_INDEX_PREFIX_LEN (sizeof(HSDIR_INDEX_PREFIX) - 1)
+
+/* Prefix of the shared random value disaster mode. */
+#define HS_SRV_DISASTER_PREFIX "shared-random-disaster"
+#define HS_SRV_DISASTER_PREFIX_LEN (sizeof(HS_SRV_DISASTER_PREFIX) - 1)
+
+/* Default value of number of hsdir replicas (hsdir_n_replicas). */
+#define HS_DEFAULT_HSDIR_N_REPLICAS 2
+/* Default value of hsdir spread store (hsdir_spread_store). */
+#define HS_DEFAULT_HSDIR_SPREAD_STORE 4
+/* Default value of hsdir spread fetch (hsdir_spread_fetch). */
+#define HS_DEFAULT_HSDIR_SPREAD_FETCH 3
+
+/* The size of a legacy RENDEZVOUS1 cell which adds up to 168 bytes. It is
+ * bigger than the 84 bytes needed for version 3 so we need to pad up to that
+ * length so it is indistinguishable between versions. */
+#define HS_LEGACY_RENDEZVOUS_CELL_SIZE \
+  (REND_COOKIE_LEN + DH1024_KEY_LEN + DIGEST_LEN)
+
+/* Type of authentication key used by an introduction point. */
+typedef enum {
+  HS_AUTH_KEY_TYPE_LEGACY  = 1,
+  HS_AUTH_KEY_TYPE_ED25519 = 2,
+} hs_auth_key_type_t;
+
+/* Return value when adding an ephemeral service through the ADD_ONION
+ * control port command. Both v2 and v3 share these. */
+typedef enum {
+  RSAE_BADAUTH     = -5, /**< Invalid auth_type/auth_clients */
+  RSAE_BADVIRTPORT = -4, /**< Invalid VIRTPORT/TARGET(s) */
+  RSAE_ADDREXISTS  = -3, /**< Onion address collision */
+  RSAE_BADPRIVKEY  = -2, /**< Invalid public key */
+  RSAE_INTERNAL    = -1, /**< Internal error */
+  RSAE_OKAY        = 0   /**< Service added as expected */
+} hs_service_add_ephemeral_status_t;
+
+/* Represents the mapping from a virtual port of a rendezvous service to a
+ * real port on some IP. */
+typedef struct rend_service_port_config_t {
+  /* The incoming HS virtual port we're mapping */
+  uint16_t virtual_port;
+  /* Is this an AF_UNIX port? */
+  unsigned int is_unix_addr:1;
+  /* The outgoing TCP port to use, if !is_unix_addr */
+  uint16_t real_port;
+  /* The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */
+  tor_addr_t real_addr;
+  /* The socket path to connect to, if is_unix_addr */
+  char unix_addr[FLEXIBLE_ARRAY_MEMBER];
+} rend_service_port_config_t;
+
+void hs_init(void);
+void hs_free_all(void);
+
+void hs_cleanup_circ(circuit_t *circ);
+
+int hs_check_service_private_dir(const char *username, const char *path,
+                                 unsigned int dir_group_readable,
+                                 unsigned int create);
+int hs_get_service_max_rend_failures(void);
+
+char *hs_path_from_filename(const char *directory, const char *filename);
+void hs_build_address(const struct ed25519_public_key_t *key, uint8_t version,
+                      char *addr_out);
+int hs_address_is_valid(const char *address);
+int hs_parse_address(const char *address, struct ed25519_public_key_t *key_out,
+                     uint8_t *checksum_out, uint8_t *version_out);
+
+void hs_build_blinded_pubkey(const struct ed25519_public_key_t *pubkey,
+                             const uint8_t *secret, size_t secret_len,
+                             uint64_t time_period_num,
+                             struct ed25519_public_key_t *pubkey_out);
+void hs_build_blinded_keypair(const struct ed25519_keypair_t *kp,
+                              const uint8_t *secret, size_t secret_len,
+                              uint64_t time_period_num,
+                              struct ed25519_keypair_t *kp_out);
+int hs_service_requires_uptime_circ(const smartlist_t *ports);
+
+void rend_data_free_(rend_data_t *data);
+#define rend_data_free(data) \
+  FREE_AND_NULL(rend_data_t, rend_data_free_, (data))
+rend_data_t *rend_data_dup(const rend_data_t *data);
+rend_data_t *rend_data_client_create(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *cookie,
+                                     rend_auth_type_t auth_type);
+rend_data_t *rend_data_service_create(const char *onion_address,
+                                      const char *pk_digest,
+                                      const uint8_t *cookie,
+                                      rend_auth_type_t auth_type);
+const char *rend_data_get_address(const rend_data_t *rend_data);
+const char *rend_data_get_desc_id(const rend_data_t *rend_data,
+                                  uint8_t replica, size_t *len_out);
+const uint8_t *rend_data_get_pk_digest(const rend_data_t *rend_data,
+                                       size_t *len_out);
+
+routerstatus_t *pick_hsdir(const char *desc_id, const char *desc_id_base32);
+
+void hs_get_subcredential(const struct ed25519_public_key_t *identity_pk,
+                          const struct ed25519_public_key_t *blinded_pk,
+                          uint8_t *subcred_out);
+
+uint64_t hs_get_previous_time_period_num(time_t now);
+uint64_t hs_get_time_period_num(time_t now);
+uint64_t hs_get_next_time_period_num(time_t now);
+time_t hs_get_start_time_of_next_time_period(time_t now);
+
+link_specifier_t *hs_link_specifier_dup(const link_specifier_t *lspec);
+
+MOCK_DECL(int, hs_in_period_between_tp_and_srv,
+          (const networkstatus_t *consensus, time_t now));
+
+uint8_t *hs_get_current_srv(uint64_t time_period_num,
+                            const networkstatus_t *ns);
+uint8_t *hs_get_previous_srv(uint64_t time_period_num,
+                             const networkstatus_t *ns);
+
+void hs_build_hsdir_index(const struct ed25519_public_key_t *identity_pk,
+                          const uint8_t *srv, uint64_t period_num,
+                          uint8_t *hsdir_index_out);
+void hs_build_hs_index(uint64_t replica,
+                       const struct ed25519_public_key_t *blinded_pk,
+                       uint64_t period_num, uint8_t *hs_index_out);
+
+int32_t hs_get_hsdir_n_replicas(void);
+int32_t hs_get_hsdir_spread_fetch(void);
+int32_t hs_get_hsdir_spread_store(void);
+
+void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk,
+                              uint64_t time_period_num,
+                              int use_second_hsdir_index,
+                              int for_fetching, smartlist_t *responsible_dirs);
+routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs,
+                              const char *req_key_str);
+
+time_t hs_hsdir_requery_period(const or_options_t *options);
+time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
+                                       const char *desc_id_base32,
+                                       time_t now, int set);
+void hs_clean_last_hid_serv_requests(time_t now);
+void hs_purge_hid_serv_from_last_hid_serv_requests(const char *desc_id);
+void hs_purge_last_hid_serv_requests(void);
+
+int hs_set_conn_addr_port(const smartlist_t *ports, edge_connection_t *conn);
+
+void hs_inc_rdv_stream_counter(origin_circuit_t *circ);
+void hs_dec_rdv_stream_counter(origin_circuit_t *circ);
+
+extend_info_t *hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
+                          const struct curve25519_public_key_t *onion_key,
+                          int direct_conn);
+
+#ifdef HS_COMMON_PRIVATE
+
+STATIC void get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out);
+
+/** The period for which a hidden service directory cannot be queried for
+ * the same descriptor ID again. */
+#define REND_HID_SERV_DIR_REQUERY_PERIOD (15 * 60)
+/** Test networks generate a new consensus every 5 or 10 seconds.
+ * So allow them to requery HSDirs much faster. */
+#define REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING (5)
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC strmap_t *get_last_hid_serv_requests(void);
+STATIC uint64_t get_time_period_length(void);
+
+STATIC uint8_t *get_first_cached_disaster_srv(void);
+STATIC uint8_t *get_second_cached_disaster_srv(void);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(HS_COMMON_PRIVATE) */
+
+#endif /* !defined(TOR_HS_COMMON_H) */
diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c
new file mode 100644
index 0000000000..cb55faa9d5
--- /dev/null
+++ b/src/feature/hs/hs_config.c
@@ -0,0 +1,591 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_config.c
+ * \brief Implement hidden service configuration subsystem.
+ *
+ * \details
+ *
+ * This file has basically one main entry point: hs_config_service_all(). It
+ * takes the torrc options and configure hidden service from it. In validate
+ * mode, nothing is added to the global service list or keys are not generated
+ * nor loaded.
+ *
+ * A service is configured in two steps. It is first created using the tor
+ * options and then put in a staging list. It will stay there until
+ * hs_service_load_all_keys() is called. That function is responsible to
+ * load/generate the keys for the service in the staging list and if
+ * successful, transfert the service to the main global service list where
+ * at that point it is ready to be used.
+ *
+ * Configuration functions are per-version and there is a main generic one for
+ * every option that is common to all version (config_generic_service).
+ **/
+
+#define HS_CONFIG_PRIVATE
+
+#include "or/hs_common.h"
+#include "or/hs_config.h"
+#include "or/hs_service.h"
+#include "or/rendservice.h"
+#include "lib/encoding/confline.h"
+#include "or/or_options_st.h"
+
+/* Using the given list of services, stage them into our global state. Every
+ * service version are handled. This function can remove entries in the given
+ * service_list.
+ *
+ * Staging a service means that we take all services in service_list and we
+ * put them in the staging list (global) which acts as a temporary list that
+ * is used by the service loading key process. In other words, staging a
+ * service puts it in a list to be considered when loading the keys and then
+ * moved to the main global list. */
+static void
+stage_services(smartlist_t *service_list)
+{
+  tor_assert(service_list);
+
+  /* This is v2 specific. Trigger service pruning which will make sure the
+   * just configured services end up in the main global list. It should only
+   * be done in non validation mode because v2 subsystem handles service
+   * object differently. */
+  rend_service_prune_list();
+
+  /* Cleanup v2 service from the list, we don't need those object anymore
+   * because we validated them all against the others and we want to stage
+   * only >= v3 service. And remember, v2 has a different object type which is
+   * shadow copied from an hs_service_t type. */
+  SMARTLIST_FOREACH_BEGIN(service_list, hs_service_t *, s) {
+    if (s->config.version == HS_VERSION_TWO) {
+      SMARTLIST_DEL_CURRENT(service_list, s);
+      hs_service_free(s);
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+  /* This is >= v3 specific. Using the newly configured service list, stage
+   * them into our global state. Every object ownership is lost after. */
+  hs_service_stage_services(service_list);
+}
+
+/* Validate the given service against all service in the given list. If the
+ * service is ephemeral, this function ignores it. Services with the same
+ * directory path aren't allowed and will return an error. If a duplicate is
+ * found, 1 is returned else 0 if none found. */
+static int
+service_is_duplicate_in_list(const smartlist_t *service_list,
+                             const hs_service_t *service)
+{
+  int ret = 0;
+
+  tor_assert(service_list);
+  tor_assert(service);
+
+  /* Ephemeral service don't have a directory configured so no need to check
+   * for a service in the list having the same path. */
+  if (service->config.is_ephemeral) {
+    goto end;
+  }
+
+  /* XXX: Validate if we have any service that has the given service dir path.
+   * This has two problems:
+   *
+   * a) It's O(n^2), but the same comment from the bottom of
+   *    rend_config_services() should apply.
+   *
+   * b) We only compare directory paths as strings, so we can't
+   *    detect two distinct paths that specify the same directory
+   *    (which can arise from symlinks, case-insensitivity, bind
+   *    mounts, etc.).
+   *
+   * It also can't detect that two separate Tor instances are trying
+   * to use the same HiddenServiceDir; for that, we would need a
+   * lock file.  But this is enough to detect a simple mistake that
+   * at least one person has actually made. */
+  SMARTLIST_FOREACH_BEGIN(service_list, const hs_service_t *, s) {
+    if (!strcmp(s->config.directory_path, service->config.directory_path)) {
+      log_warn(LD_REND, "Another hidden service is already configured "
+                        "for directory %s",
+               escaped(service->config.directory_path));
+      ret = 1;
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+ end:
+  return ret;
+}
+
+/* Helper function: Given an configuration option name, its value, a minimum
+ * min and a maxium max, parse the value as a uint64_t. On success, ok is set
+ * to 1 and ret is the parsed value. On error, ok is set to 0 and ret must be
+ * ignored. This function logs both on error and success. */
+static uint64_t
+helper_parse_uint64(const char *opt, const char *value, uint64_t min,
+                    uint64_t max, int *ok)
+{
+  uint64_t ret = 0;
+
+  tor_assert(opt);
+  tor_assert(value);
+  tor_assert(ok);
+
+  *ok = 0;
+  ret = tor_parse_uint64(value, 10, min, max, ok, NULL);
+  if (!*ok) {
+    log_warn(LD_CONFIG, "%s must be between %" PRIu64 " and %"PRIu64
+                        ", not %s.",
+             opt, min, max, value);
+    goto err;
+  }
+  log_info(LD_CONFIG, "%s was parsed to %" PRIu64, opt, ret);
+ err:
+  return ret;
+}
+
+/* Return true iff the given options starting at line_ for a hidden service
+ * contains at least one invalid option. Each hidden service option don't
+ * apply to all versions so this function can find out. The line_ MUST start
+ * right after the HiddenServiceDir line of this service.
+ *
+ * This is mainly for usability so we can inform the user of any invalid
+ * option for the hidden service version instead of silently ignoring. */
+static int
+config_has_invalid_options(const config_line_t *line_,
+                           const hs_service_t *service)
+{
+  int ret = 0;
+  const char **optlist;
+  const config_line_t *line;
+
+  tor_assert(service);
+  tor_assert(service->config.version <= HS_VERSION_MAX);
+
+  /* List of options that a v3 service doesn't support thus must exclude from
+   * its configuration. */
+  const char *opts_exclude_v3[] = {
+    "HiddenServiceAuthorizeClient",
+    NULL /* End marker. */
+  };
+
+  /* Defining the size explicitly allows us to take advantage of the compiler
+   * which warns us if we ever bump the max version but forget to grow this
+   * array. The plus one is because we have a version 0 :). */
+  struct {
+    const char **list;
+  } exclude_lists[HS_VERSION_MAX + 1] = {
+    { NULL }, /* v0. */
+    { NULL }, /* v1. */
+    { NULL }, /* v2 */
+    { opts_exclude_v3 }, /* v3. */
+  };
+
+  optlist = exclude_lists[service->config.version].list;
+  if (optlist == NULL) {
+    /* No exclude options to look at for this version. */
+    goto end;
+  }
+  for (int i = 0; optlist[i]; i++) {
+    const char *opt = optlist[i];
+    for (line = line_; line; line = line->next) {
+      if (!strcasecmp(line->key, "HiddenServiceDir")) {
+        /* We just hit the next hidden service, stop right now. */
+        goto end;
+      }
+      if (!strcasecmp(line->key, opt)) {
+        log_warn(LD_CONFIG, "Hidden service option %s is incompatible with "
+                            "version %" PRIu32 " of service in %s",
+                 opt, service->config.version,
+                 service->config.directory_path);
+        ret = 1;
+        /* Continue the loop so we can find all possible options. */
+        continue;
+      }
+    }
+  }
+ end:
+  return ret;
+}
+
+/* Validate service configuration. This is used when loading the configuration
+ * and once we've setup a service object, it's config object is passed to this
+ * function for further validation. This does not validate service key
+ * material. Return 0 if valid else -1 if invalid. */
+static int
+config_validate_service(const hs_service_config_t *config)
+{
+  tor_assert(config);
+
+  /* Amount of ports validation. */
+  if (!config->ports || smartlist_len(config->ports) == 0) {
+    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured.",
+             escaped(config->directory_path));
+    goto invalid;
+  }
+
+  /* Valid. */
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* Configuration funcion for a version 3 service. The line_ must be pointing
+ * to the directive directly after a HiddenServiceDir. That way, when hitting
+ * the next HiddenServiceDir line or reaching the end of the list of lines, we
+ * know that we have to stop looking for more options. The given service
+ * object must be already allocated and passed through
+ * config_generic_service() prior to calling this function.
+ *
+ * Return 0 on success else a negative value. */
+static int
+config_service_v3(const config_line_t *line_,
+                  hs_service_config_t *config)
+{
+  int have_num_ip = 0;
+  const char *dup_opt_seen = NULL;
+  const config_line_t *line;
+
+  tor_assert(config);
+
+  for (line = line_; line; line = line->next) {
+    int ok = 0;
+    if (!strcasecmp(line->key, "HiddenServiceDir")) {
+      /* We just hit the next hidden service, stop right now. */
+      break;
+    }
+    /* Number of introduction points. */
+    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
+      config->num_intro_points =
+        (unsigned int) helper_parse_uint64(line->key, line->value,
+                                           NUM_INTRO_POINTS_DEFAULT,
+                                           HS_CONFIG_V3_MAX_INTRO_POINTS,
+                                           &ok);
+      if (!ok || have_num_ip) {
+        if (have_num_ip)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_num_ip = 1;
+      continue;
+    }
+  }
+
+  /* We do not load the key material for the service at this stage. This is
+   * done later once tor can confirm that it is in a running state. */
+
+  /* We are about to return a fully configured service so do one last pass of
+   * validation at it. */
+  if (config_validate_service(config) < 0) {
+    goto err;
+  }
+
+  return 0;
+ err:
+  if (dup_opt_seen) {
+    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
+  }
+  return -1;
+}
+
+/* Configure a service using the given options in line_ and options. This is
+ * called for any service regardless of its version which means that all
+ * directives in this function are generic to any service version. This
+ * function will also check the validity of the service directory path.
+ *
+ * The line_ must be pointing to the directive directly after a
+ * HiddenServiceDir. That way, when hitting the next HiddenServiceDir line or
+ * reaching the end of the list of lines, we know that we have to stop looking
+ * for more options.
+ *
+ * Return 0 on success else -1. */
+static int
+config_generic_service(const config_line_t *line_,
+                       const or_options_t *options,
+                       hs_service_t *service)
+{
+  int dir_seen = 0;
+  const config_line_t *line;
+  hs_service_config_t *config;
+  /* If this is set, we've seen a duplicate of this option. Keep the string
+   * so we can log the directive. */
+  const char *dup_opt_seen = NULL;
+  /* These variables will tell us if we ever have duplicate. */
+  int have_version = 0, have_allow_unknown_ports = 0;
+  int have_dir_group_read = 0, have_max_streams = 0;
+  int have_max_streams_close = 0;
+
+  tor_assert(line_);
+  tor_assert(options);
+  tor_assert(service);
+
+  /* Makes thing easier. */
+  config = &service->config;
+
+  /* The first line starts with HiddenServiceDir so we consider what's next is
+   * the configuration of the service. */
+  for (line = line_; line ; line = line->next) {
+    int ok = 0;
+
+    /* This indicate that we have a new service to configure. */
+    if (!strcasecmp(line->key, "HiddenServiceDir")) {
+      /* This function only configures one service at a time so if we've
+       * already seen one, stop right now. */
+      if (dir_seen) {
+        break;
+      }
+      /* Ok, we've seen one and we are about to configure it. */
+      dir_seen = 1;
+      config->directory_path = tor_strdup(line->value);
+      log_info(LD_CONFIG, "HiddenServiceDir=%s. Configuring...",
+               escaped(config->directory_path));
+      continue;
+    }
+    if (BUG(!dir_seen)) {
+      goto err;
+    }
+    /* Version of the service. */
+    if (!strcasecmp(line->key, "HiddenServiceVersion")) {
+      service->config.version =
+        (uint32_t) helper_parse_uint64(line->key, line->value, HS_VERSION_MIN,
+                                       HS_VERSION_MAX, &ok);
+      if (!ok || have_version) {
+        if (have_version)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_version = 1;
+      continue;
+    }
+    /* Virtual port. */
+    if (!strcasecmp(line->key, "HiddenServicePort")) {
+      char *err_msg = NULL;
+      /* XXX: Can we rename this? */
+      rend_service_port_config_t *portcfg =
+        rend_service_parse_port_config(line->value, " ", &err_msg);
+      if (!portcfg) {
+        if (err_msg) {
+          log_warn(LD_CONFIG, "%s", err_msg);
+        }
+        tor_free(err_msg);
+        goto err;
+      }
+      tor_assert(!err_msg);
+      smartlist_add(config->ports, portcfg);
+      log_info(LD_CONFIG, "HiddenServicePort=%s for %s",
+               line->value, escaped(config->directory_path));
+      continue;
+    }
+    /* Do we allow unknown ports. */
+    if (!strcasecmp(line->key, "HiddenServiceAllowUnknownPorts")) {
+      config->allow_unknown_ports =
+        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
+      if (!ok || have_allow_unknown_ports) {
+        if (have_allow_unknown_ports)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_allow_unknown_ports = 1;
+      continue;
+    }
+    /* Directory group readable. */
+    if (!strcasecmp(line->key, "HiddenServiceDirGroupReadable")) {
+      config->dir_group_readable =
+        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
+      if (!ok || have_dir_group_read) {
+        if (have_dir_group_read)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_dir_group_read = 1;
+      continue;
+    }
+    /* Maximum streams per circuit. */
+    if (!strcasecmp(line->key, "HiddenServiceMaxStreams")) {
+      config->max_streams_per_rdv_circuit =
+        helper_parse_uint64(line->key, line->value, 0,
+                            HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT, &ok);
+      if (!ok || have_max_streams) {
+        if (have_max_streams)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_max_streams = 1;
+      continue;
+    }
+    /* Maximum amount of streams before we close the circuit. */
+    if (!strcasecmp(line->key, "HiddenServiceMaxStreamsCloseCircuit")) {
+      config->max_streams_close_circuit =
+        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
+      if (!ok || have_max_streams_close) {
+        if (have_max_streams_close)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_max_streams_close = 1;
+      continue;
+    }
+  }
+
+  /* Check if we are configured in non anonymous mode meaning every service
+   * becomes a single onion service. */
+  if (rend_service_non_anonymous_mode_enabled(options)) {
+    config->is_single_onion = 1;
+    /* We will add support for IPv6-only v3 single onion services in a future
+     * Tor version. This won't catch "ReachableAddresses reject *4", but that
+     * option doesn't work anyway. */
+    if (options->ClientUseIPv4 == 0 && config->version == HS_VERSION_THREE) {
+      log_warn(LD_CONFIG, "IPv6-only v3 single onion services are not "
+               "supported. Set HiddenServiceSingleHopMode 0 and "
+               "HiddenServiceNonAnonymousMode 0, or set ClientUseIPv4 1.");
+      goto err;
+    }
+  }
+
+  /* Success */
+  return 0;
+ err:
+  if (dup_opt_seen) {
+    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
+  }
+  return -1;
+}
+
+/* Configure a service using the given line and options. This function will
+ * call the corresponding configuration function for a specific service
+ * version and validate the service against the other ones. On success, add
+ * the service to the given list and return 0. On error, nothing is added to
+ * the list and a negative value is returned. */
+static int
+config_service(const config_line_t *line, const or_options_t *options,
+               smartlist_t *service_list)
+{
+  int ret;
+  hs_service_t *service = NULL;
+
+  tor_assert(line);
+  tor_assert(options);
+  tor_assert(service_list);
+
+  /* We have a new hidden service. */
+  service = hs_service_new(options);
+  /* We'll configure that service as a generic one and then pass it to a
+   * specific function according to the configured version number. */
+  if (config_generic_service(line, options, service) < 0) {
+    goto err;
+  }
+  tor_assert(service->config.version <= HS_VERSION_MAX);
+  /* Before we configure the service on a per-version basis, we'll make
+   * sure that this set of options for a service are valid that is for
+   * instance an option only for v2 is not used for v3. */
+  if (config_has_invalid_options(line->next, service)) {
+    goto err;
+  }
+  /* Check permission on service directory that was just parsed. And this must
+   * be done regardless of the service version. Do not ask for the directory
+   * to be created, this is done when the keys are loaded because we could be
+   * in validation mode right now. */
+  if (hs_check_service_private_dir(options->User,
+                                   service->config.directory_path,
+                                   service->config.dir_group_readable,
+                                   0) < 0) {
+    goto err;
+  }
+  /* Different functions are in charge of specific options for a version. We
+   * start just after the service directory line so once we hit another
+   * directory line, the function knows that it has to stop parsing. */
+  switch (service->config.version) {
+  case HS_VERSION_TWO:
+    ret = rend_config_service(line->next, options, &service->config);
+    break;
+  case HS_VERSION_THREE:
+    ret = config_service_v3(line->next, &service->config);
+    break;
+  default:
+    /* We do validate before if we support the parsed version. */
+    tor_assert_nonfatal_unreached();
+    goto err;
+  }
+  if (ret < 0) {
+    goto err;
+  }
+  /* We'll check if this service can be kept depending on the others
+   * configured previously. */
+  if (service_is_duplicate_in_list(service_list, service)) {
+    goto err;
+  }
+  /* Passes, add it to the given list. */
+  smartlist_add(service_list, service);
+  return 0;
+
+ err:
+  hs_service_free(service);
+  return -1;
+}
+
+/* From a set of <b>options</b>, setup every hidden service found. Return 0 on
+ * success or -1 on failure. If <b>validate_only</b> is set, parse, warn and
+ * return as normal, but don't actually change the configured services. */
+int
+hs_config_service_all(const or_options_t *options, int validate_only)
+{
+  int dir_option_seen = 0, ret = -1;
+  const config_line_t *line;
+  smartlist_t *new_service_list = NULL;
+
+  tor_assert(options);
+
+  /* Newly configured service are put in that list which is then used for
+   * validation and staging for >= v3. */
+  new_service_list = smartlist_new();
+
+  for (line = options->RendConfigLines; line; line = line->next) {
+    /* Ignore all directives that aren't the start of a service. */
+    if (strcasecmp(line->key, "HiddenServiceDir")) {
+      if (!dir_option_seen) {
+        log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive",
+                 line->key);
+        goto err;
+      }
+      continue;
+    }
+    /* Flag that we've seen a directory directive and we'll use it to make
+     * sure that the torrc options ordering is actually valid. */
+    dir_option_seen = 1;
+
+    /* Try to configure this service now. On success, it will be added to the
+     * list and validated against the service in that same list. */
+    if (config_service(line, options, new_service_list) < 0) {
+      goto err;
+    }
+  }
+
+  /* In non validation mode, we'll stage those services we just successfully
+   * configured. Service ownership is transferred from the list to the global
+   * state. If any service is invalid, it will be removed from the list and
+   * freed. All versions are handled in that function. */
+  if (!validate_only) {
+    stage_services(new_service_list);
+  } else {
+    /* We've just validated that we were able to build a clean working list of
+     * services. We don't need those objects anymore. */
+    SMARTLIST_FOREACH(new_service_list, hs_service_t *, s,
+                      hs_service_free(s));
+    /* For the v2 subsystem, the configuration function adds the service
+     * object to the staging list and it is transferred in the main list
+     * through the prunning process. In validation mode, we thus have to purge
+     * the staging list so it's not kept in memory as valid service. */
+    rend_service_free_staging_list();
+  }
+
+  /* Success. Note that the service list has no ownership of its content. */
+  ret = 0;
+  goto end;
+
+ err:
+  SMARTLIST_FOREACH(new_service_list, hs_service_t *, s, hs_service_free(s));
+
+ end:
+  smartlist_free(new_service_list);
+  /* Tor main should call the free all function on error. */
+  return ret;
+}
diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h
new file mode 100644
index 0000000000..461d58d384
--- /dev/null
+++ b/src/feature/hs/hs_config.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_config.h
+ * \brief Header file containing configuration ABI/API for the HS subsytem.
+ **/
+
+#ifndef TOR_HS_CONFIG_H
+#define TOR_HS_CONFIG_H
+
+#include "or/or.h"
+
+/* Max value for HiddenServiceMaxStreams */
+#define HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT 65535
+/* Maximum number of intro points per version 3 services. */
+#define HS_CONFIG_V3_MAX_INTRO_POINTS 20
+
+/* API */
+
+int hs_config_service_all(const or_options_t *options, int validate_only);
+
+#endif /* !defined(TOR_HS_CONFIG_H) */
+
diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c
new file mode 100644
index 0000000000..be456e8da4
--- /dev/null
+++ b/src/feature/hs/hs_control.c
@@ -0,0 +1,261 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_control.c
+ * \brief Contains control port event related code.
+ **/
+
+#include "or/or.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/hs_common.h"
+#include "or/hs_control.h"
+#include "or/hs_descriptor.h"
+#include "or/hs_service.h"
+#include "or/nodelist.h"
+
+#include "or/node_st.h"
+#include "or/routerstatus_st.h"
+
+/* Send on the control port the "HS_DESC REQUESTED [...]" event.
+ *
+ * The onion_pk is the onion service public key, base64_blinded_pk is the
+ * base64 encoded blinded key for the service and hsdir_rs is the routerstatus
+ * object of the HSDir that this request is for. */
+void
+hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk,
+                                const char *base64_blinded_pk,
+                                const routerstatus_t *hsdir_rs)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  const uint8_t *hsdir_index;
+  const node_t *hsdir_node;
+
+  tor_assert(onion_pk);
+  tor_assert(base64_blinded_pk);
+  tor_assert(hsdir_rs);
+
+  hs_build_address(onion_pk, HS_VERSION_THREE, onion_address);
+
+  /* Get the node from the routerstatus object to get the HSDir index used for
+   * this request. We can't have a routerstatus entry without a node and we
+   * can't pick a node without an hsdir_index. */
+  hsdir_node = node_get_by_id(hsdir_rs->identity_digest);
+  tor_assert(hsdir_node);
+  /* This is a fetch event. */
+  hsdir_index = hsdir_node->hsdir_index.fetch;
+
+  /* Trigger the event. */
+  control_event_hs_descriptor_requested(onion_address, REND_NO_AUTH,
+                                        hsdir_rs->identity_digest,
+                                        base64_blinded_pk,
+                                        hex_str((const char *) hsdir_index,
+                                                DIGEST256_LEN));
+  memwipe(onion_address, 0, sizeof(onion_address));
+}
+
+/* Send on the control port the "HS_DESC FAILED [...]" event.
+ *
+ * Using a directory connection identifier, the HSDir identity digest and a
+ * reason for the failure. None can be NULL. */
+void
+hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
+                             const char *hsdir_id_digest,
+                             const char *reason)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+  tor_assert(reason);
+
+  /* Build onion address and encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
+                                       &ident->blinded_pk) < 0) {
+    return;
+  }
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hsv3_descriptor_failed(onion_address, base64_blinded_pk,
+                                       hsdir_id_digest, reason);
+}
+
+/* Send on the control port the "HS_DESC RECEIVED [...]" event.
+ *
+ * Using a directory connection identifier and the HSDir identity digest.
+ * None can be NULL. */
+void
+hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
+                               const char *hsdir_id_digest)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+
+  /* Build onion address and encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
+                                       &ident->blinded_pk) < 0) {
+    return;
+  }
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hsv3_descriptor_received(onion_address, base64_blinded_pk,
+                                         hsdir_id_digest);
+}
+
+/* Send on the control port the "HS_DESC CREATED [...]" event.
+ *
+ * Using the onion address of the descriptor's service and the blinded public
+ * key of the descriptor as a descriptor ID. None can be NULL. */
+void
+hs_control_desc_event_created(const char *onion_address,
+                              const ed25519_public_key_t *blinded_pk)
+{
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(onion_address);
+  tor_assert(blinded_pk);
+
+  /* Build base64 encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
+    return;
+  }
+
+  /* Version 3 doesn't use the replica number in its descriptor ID computation
+   * so we pass negative value so the control port subsystem can ignore it. */
+  control_event_hs_descriptor_created(onion_address, base64_blinded_pk, -1);
+}
+
+/* Send on the control port the "HS_DESC UPLOAD [...]" event.
+ *
+ * Using the onion address of the descriptor's service, the HSDir identity
+ * digest, the blinded public key of the descriptor as a descriptor ID and the
+ * HSDir index for this particular request. None can be NULL. */
+void
+hs_control_desc_event_upload(const char *onion_address,
+                             const char *hsdir_id_digest,
+                             const ed25519_public_key_t *blinded_pk,
+                             const uint8_t *hsdir_index)
+{
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(onion_address);
+  tor_assert(hsdir_id_digest);
+  tor_assert(blinded_pk);
+  tor_assert(hsdir_index);
+
+  /* Build base64 encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
+    return;
+  }
+
+  control_event_hs_descriptor_upload(onion_address, hsdir_id_digest,
+                                     base64_blinded_pk,
+                                     hex_str((const char *) hsdir_index,
+                                             DIGEST256_LEN));
+}
+
+/* Send on the control port the "HS_DESC UPLOADED [...]" event.
+ *
+ * Using the directory connection identifier and the HSDir identity digest.
+ * None can be NULL. */
+void
+hs_control_desc_event_uploaded(const hs_ident_dir_conn_t *ident,
+                               const char *hsdir_id_digest)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hs_descriptor_uploaded(hsdir_id_digest, onion_address);
+}
+
+/* Send on the control port the "HS_DESC_CONTENT [...]" event.
+ *
+ * Using the directory connection identifier, the HSDir identity digest and
+ * the body of the descriptor (as it was received from the directory). None
+ * can be NULL. */
+void
+hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
+                              const char *hsdir_id_digest,
+                              const char *body)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+
+  /* Build onion address and encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
+                                       &ident->blinded_pk) < 0) {
+    return;
+  }
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hs_descriptor_content(onion_address, base64_blinded_pk,
+                                      hsdir_id_digest, body);
+}
+
+/* Handle the "HSPOST [...]" command. The body is an encoded descriptor for
+ * the given onion_address. The descriptor will be uploaded to each directory
+ * in hsdirs_rs. If NULL, the responsible directories for the current time
+ * period will be selected.
+ *
+ * Return -1 on if the descriptor plaintext section is not decodable. Else, 0
+ * on success. */
+int
+hs_control_hspost_command(const char *body, const char *onion_address,
+                          const smartlist_t *hsdirs_rs)
+{
+  int ret = -1;
+  ed25519_public_key_t identity_pk;
+  hs_desc_plaintext_data_t plaintext;
+  smartlist_t *hsdirs = NULL;
+
+  tor_assert(body);
+  tor_assert(onion_address);
+
+  /* This can't fail because we require the caller to pass us a valid onion
+   * address that has passed hs_address_is_valid(). */
+  if (BUG(hs_parse_address(onion_address, &identity_pk, NULL, NULL) < 0)) {
+    goto done; // LCOV_EXCL_LINE
+  }
+
+  /* Only decode the plaintext part which is what the directory will do to
+   * validate before caching. */
+  if (hs_desc_decode_plaintext(body, &plaintext) < 0) {
+    goto done;
+  }
+
+  /* No HSDir(s) given, we'll compute what the current ones should be. */
+  if (hsdirs_rs == NULL) {
+    hsdirs = smartlist_new();
+    hs_get_responsible_hsdirs(&plaintext.blinded_pubkey,
+                              hs_get_time_period_num(0),
+                              0, /* Always the current descriptor which uses
+                                  * the first hsdir index. */
+                              0, /* It is for storing on a directory. */
+                              hsdirs);
+    hsdirs_rs = hsdirs;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(hsdirs_rs, const routerstatus_t *, rs) {
+    hs_service_upload_desc_to_dir(body, plaintext.version, &identity_pk,
+                                  &plaintext.blinded_pubkey, rs);
+  } SMARTLIST_FOREACH_END(rs);
+  ret = 0;
+
+ done:
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(hsdirs);
+  return ret;
+}
diff --git a/src/feature/hs/hs_control.h b/src/feature/hs/hs_control.h
new file mode 100644
index 0000000000..040ce4a793
--- /dev/null
+++ b/src/feature/hs/hs_control.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_control.h
+ * \brief Header file containing control port event related code.
+ **/
+
+#ifndef TOR_HS_CONTROL_H
+#define TOR_HS_CONTROL_H
+
+#include "or/hs_ident.h"
+
+/* Event "HS_DESC REQUESTED [...]" */
+void hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk,
+                                     const char *base64_blinded_pk,
+                                     const routerstatus_t *hsdir_rs);
+
+/* Event "HS_DESC FAILED [...]" */
+void hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
+                                  const char *hsdir_id_digest,
+                                  const char *reason);
+
+/* Event "HS_DESC RECEIVED [...]" */
+void hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
+                                    const char *hsdir_id_digest);
+
+/* Event "HS_DESC CREATED [...]" */
+void hs_control_desc_event_created(const char *onion_address,
+                                   const ed25519_public_key_t *blinded_pk);
+
+/* Event "HS_DESC UPLOAD [...]" */
+void hs_control_desc_event_upload(const char *onion_address,
+                                  const char *hsdir_id_digest,
+                                  const ed25519_public_key_t *blinded_pk,
+                                  const uint8_t *hsdir_index);
+
+/* Event "HS_DESC UPLOADED [...]" */
+void hs_control_desc_event_uploaded(const hs_ident_dir_conn_t *ident,
+                                    const char *hsdir_id_digest);
+
+/* Event "HS_DESC_CONTENT [...]" */
+void hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
+                                   const char *hsdir_id_digest,
+                                   const char *body);
+
+/* Command "HSPOST [...]" */
+int hs_control_hspost_command(const char *body, const char *onion_address,
+                              const smartlist_t *hsdirs_rs);
+
+#endif /* !defined(TOR_HS_CONTROL_H) */
+
diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c
new file mode 100644
index 0000000000..5fd8971dc0
--- /dev/null
+++ b/src/feature/hs/hs_descriptor.c
@@ -0,0 +1,2611 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_descriptor.c
+ * \brief Handle hidden service descriptor encoding/decoding.
+ *
+ * \details
+ * Here is a graphical depiction of an HS descriptor and its layers:
+ *
+ *      +------------------------------------------------------+
+ *      |DESCRIPTOR HEADER:                                    |
+ *      |  hs-descriptor 3                                     |
+ *      |  descriptor-lifetime 180                             |
+ *      |  ...                                                 |
+ *      |  superencrypted                                      |
+ *      |+---------------------------------------------------+ |
+ *      ||SUPERENCRYPTED LAYER (aka OUTER ENCRYPTED LAYER):  | |
+ *      ||  desc-auth-type x25519                            | |
+ *      ||  desc-auth-ephemeral-key                          | |
+ *      ||  auth-client                                      | |
+ *      ||  auth-client                                      | |
+ *      ||  ...                                              | |
+ *      ||  encrypted                                        | |
+ *      ||+-------------------------------------------------+| |
+ *      |||ENCRYPTED LAYER (aka INNER ENCRYPTED LAYER):     || |
+ *      |||  create2-formats                                || |
+ *      |||  intro-auth-required                            || |
+ *      |||  introduction-point                             || |
+ *      |||  introduction-point                             || |
+ *      |||  ...                                            || |
+ *      ||+-------------------------------------------------+| |
+ *      |+---------------------------------------------------+ |
+ *      +------------------------------------------------------+
+ *
+ * The DESCRIPTOR HEADER section is completely unencrypted and contains generic
+ * descriptor metadata.
+ *
+ * The SUPERENCRYPTED LAYER section is the first layer of encryption, and it's
+ * encrypted using the blinded public key of the hidden service to protect
+ * against entities who don't know its onion address. The clients of the hidden
+ * service know its onion address and blinded public key, whereas third-parties
+ * (like HSDirs) don't know it (except if it's a public hidden service).
+ *
+ * The ENCRYPTED LAYER section is the second layer of encryption, and it's
+ * encrypted using the client authorization key material (if those exist). When
+ * client authorization is enabled, this second layer of encryption protects
+ * the descriptor content from unauthorized entities. If client authorization
+ * is disabled, this second layer of encryption does not provide any extra
+ * security but is still present. The plaintext of this layer contains all the
+ * information required to connect to the hidden service like its list of
+ * introduction points.
+ **/
+
+/* For unit tests.*/
+#define HS_DESCRIPTOR_PRIVATE
+
+#include "or/or.h"
+#include "trunnel/ed25519_cert.h" /* Trunnel interface. */
+#include "or/hs_descriptor.h"
+#include "or/circuitbuild.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/parsecommon.h"
+#include "or/rendcache.h"
+#include "or/hs_cache.h"
+#include "or/hs_config.h"
+#include "or/torcert.h" /* tor_cert_encode_ed22519() */
+#include "lib/memarea/memarea.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+#include "or/extend_info_st.h"
+
+/* Constant string value used for the descriptor format. */
+#define str_hs_desc "hs-descriptor"
+#define str_desc_cert "descriptor-signing-key-cert"
+#define str_rev_counter "revision-counter"
+#define str_superencrypted "superencrypted"
+#define str_encrypted "encrypted"
+#define str_signature "signature"
+#define str_lifetime "descriptor-lifetime"
+/* Constant string value for the encrypted part of the descriptor. */
+#define str_create2_formats "create2-formats"
+#define str_intro_auth_required "intro-auth-required"
+#define str_single_onion "single-onion-service"
+#define str_intro_point "introduction-point"
+#define str_ip_onion_key "onion-key"
+#define str_ip_auth_key "auth-key"
+#define str_ip_enc_key "enc-key"
+#define str_ip_enc_key_cert "enc-key-cert"
+#define str_ip_legacy_key "legacy-key"
+#define str_ip_legacy_key_cert "legacy-key-cert"
+#define str_intro_point_start "\n" str_intro_point " "
+/* Constant string value for the construction to encrypt the encrypted data
+ * section. */
+#define str_enc_const_superencryption "hsdir-superencrypted-data"
+#define str_enc_const_encryption "hsdir-encrypted-data"
+/* Prefix required to compute/verify HS desc signatures */
+#define str_desc_sig_prefix "Tor onion service descriptor sig v3"
+#define str_desc_auth_type "desc-auth-type"
+#define str_desc_auth_key "desc-auth-ephemeral-key"
+#define str_desc_auth_client "auth-client"
+#define str_encrypted "encrypted"
+
+/* Authentication supported types. */
+static const struct {
+  hs_desc_auth_type_t type;
+  const char *identifier;
+} intro_auth_types[] = {
+  { HS_DESC_AUTH_ED25519, "ed25519" },
+  /* Indicate end of array. */
+  { 0, NULL }
+};
+
+/* Descriptor ruleset. */
+static token_rule_t hs_desc_v3_token_table[] = {
+  T1_START(str_hs_desc, R_HS_DESCRIPTOR, EQ(1), NO_OBJ),
+  T1(str_lifetime, R3_DESC_LIFETIME, EQ(1), NO_OBJ),
+  T1(str_desc_cert, R3_DESC_SIGNING_CERT, NO_ARGS, NEED_OBJ),
+  T1(str_rev_counter, R3_REVISION_COUNTER, EQ(1), NO_OBJ),
+  T1(str_superencrypted, R3_SUPERENCRYPTED, NO_ARGS, NEED_OBJ),
+  T1_END(str_signature, R3_SIGNATURE, EQ(1), NO_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the superencrypted section. */
+static token_rule_t hs_desc_superencrypted_v3_token_table[] = {
+  T1_START(str_desc_auth_type, R3_DESC_AUTH_TYPE, GE(1), NO_OBJ),
+  T1(str_desc_auth_key, R3_DESC_AUTH_KEY, GE(1), NO_OBJ),
+  T1N(str_desc_auth_client, R3_DESC_AUTH_CLIENT, GE(3), NO_OBJ),
+  T1(str_encrypted, R3_ENCRYPTED, NO_ARGS, NEED_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the encrypted section. */
+static token_rule_t hs_desc_encrypted_v3_token_table[] = {
+  T1_START(str_create2_formats, R3_CREATE2_FORMATS, CONCAT_ARGS, NO_OBJ),
+  T01(str_intro_auth_required, R3_INTRO_AUTH_REQUIRED, ARGS, NO_OBJ),
+  T01(str_single_onion, R3_SINGLE_ONION_SERVICE, ARGS, NO_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the introduction points section. */
+static token_rule_t hs_desc_intro_point_v3_token_table[] = {
+  T1_START(str_intro_point, R3_INTRODUCTION_POINT, EQ(1), NO_OBJ),
+  T1N(str_ip_onion_key, R3_INTRO_ONION_KEY, GE(2), OBJ_OK),
+  T1(str_ip_auth_key, R3_INTRO_AUTH_KEY, NO_ARGS, NEED_OBJ),
+  T1(str_ip_enc_key, R3_INTRO_ENC_KEY, GE(2), OBJ_OK),
+  T1(str_ip_enc_key_cert, R3_INTRO_ENC_KEY_CERT, ARGS, OBJ_OK),
+  T01(str_ip_legacy_key, R3_INTRO_LEGACY_KEY, ARGS, NEED_KEY_1024),
+  T01(str_ip_legacy_key_cert, R3_INTRO_LEGACY_KEY_CERT, ARGS, OBJ_OK),
+  END_OF_TABLE
+};
+
+/* Free the content of the plaintext section of a descriptor. */
+STATIC void
+desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  if (desc->superencrypted_blob) {
+    tor_free(desc->superencrypted_blob);
+  }
+  tor_cert_free(desc->signing_key_cert);
+
+  memwipe(desc, 0, sizeof(*desc));
+}
+
+/* Free the content of the encrypted section of a descriptor. */
+static void
+desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  if (desc->intro_auth_types) {
+    SMARTLIST_FOREACH(desc->intro_auth_types, char *, a, tor_free(a));
+    smartlist_free(desc->intro_auth_types);
+  }
+  if (desc->intro_points) {
+    SMARTLIST_FOREACH(desc->intro_points, hs_desc_intro_point_t *, ip,
+                      hs_desc_intro_point_free(ip));
+    smartlist_free(desc->intro_points);
+  }
+  memwipe(desc, 0, sizeof(*desc));
+}
+
+/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out.
+ * We use SHA3-256 for the MAC computation.
+ * This function can't fail. */
+static void
+build_mac(const uint8_t *mac_key, size_t mac_key_len,
+          const uint8_t *salt, size_t salt_len,
+          const uint8_t *encrypted, size_t encrypted_len,
+          uint8_t *mac_out, size_t mac_len)
+{
+  crypto_digest_t *digest;
+
+  const uint64_t mac_len_netorder = tor_htonll(mac_key_len);
+  const uint64_t salt_len_netorder = tor_htonll(salt_len);
+
+  tor_assert(mac_key);
+  tor_assert(salt);
+  tor_assert(encrypted);
+  tor_assert(mac_out);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  /* As specified in section 2.5 of proposal 224, first add the mac key
+   * then add the salt first and then the encrypted section. */
+
+  crypto_digest_add_bytes(digest, (const char *) &mac_len_netorder, 8);
+  crypto_digest_add_bytes(digest, (const char *) mac_key, mac_key_len);
+  crypto_digest_add_bytes(digest, (const char *) &salt_len_netorder, 8);
+  crypto_digest_add_bytes(digest, (const char *) salt, salt_len);
+  crypto_digest_add_bytes(digest, (const char *) encrypted, encrypted_len);
+  crypto_digest_get_digest(digest, (char *) mac_out, mac_len);
+  crypto_digest_free(digest);
+}
+
+/* Using a given decriptor object, build the secret input needed for the
+ * KDF and put it in the dst pointer which is an already allocated buffer
+ * of size dstlen. */
+static void
+build_secret_input(const hs_descriptor_t *desc, uint8_t *dst, size_t dstlen)
+{
+  size_t offset = 0;
+
+  tor_assert(desc);
+  tor_assert(dst);
+  tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN <= dstlen);
+
+  /* XXX use the destination length as the memcpy length */
+  /* Copy blinded public key. */
+  memcpy(dst, desc->plaintext_data.blinded_pubkey.pubkey,
+         sizeof(desc->plaintext_data.blinded_pubkey.pubkey));
+  offset += sizeof(desc->plaintext_data.blinded_pubkey.pubkey);
+  /* Copy subcredential. */
+  memcpy(dst + offset, desc->subcredential, sizeof(desc->subcredential));
+  offset += sizeof(desc->subcredential);
+  /* Copy revision counter value. */
+  set_uint64(dst + offset, tor_htonll(desc->plaintext_data.revision_counter));
+  offset += sizeof(uint64_t);
+  tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN == offset);
+}
+
+/* Do the KDF construction and put the resulting data in key_out which is of
+ * key_out_len length. It uses SHAKE-256 as specified in the spec. */
+static void
+build_kdf_key(const hs_descriptor_t *desc,
+              const uint8_t *salt, size_t salt_len,
+              uint8_t *key_out, size_t key_out_len,
+              int is_superencrypted_layer)
+{
+  uint8_t secret_input[HS_DESC_ENCRYPTED_SECRET_INPUT_LEN];
+  crypto_xof_t *xof;
+
+  tor_assert(desc);
+  tor_assert(salt);
+  tor_assert(key_out);
+
+  /* Build the secret input for the KDF computation. */
+  build_secret_input(desc, secret_input, sizeof(secret_input));
+
+  xof = crypto_xof_new();
+  /* Feed our KDF. [SHAKE it like a polaroid picture --Yawning]. */
+  crypto_xof_add_bytes(xof, secret_input, sizeof(secret_input));
+  crypto_xof_add_bytes(xof, salt, salt_len);
+
+  /* Feed in the right string constant based on the desc layer */
+  if (is_superencrypted_layer) {
+    crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_superencryption,
+                         strlen(str_enc_const_superencryption));
+  } else {
+    crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_encryption,
+                         strlen(str_enc_const_encryption));
+  }
+
+  /* Eat from our KDF. */
+  crypto_xof_squeeze_bytes(xof, key_out, key_out_len);
+  crypto_xof_free(xof);
+  memwipe(secret_input,  0, sizeof(secret_input));
+}
+
+/* Using the given descriptor and salt, run it through our KDF function and
+ * then extract a secret key in key_out, the IV in iv_out and MAC in mac_out.
+ * This function can't fail. */
+static void
+build_secret_key_iv_mac(const hs_descriptor_t *desc,
+                        const uint8_t *salt, size_t salt_len,
+                        uint8_t *key_out, size_t key_len,
+                        uint8_t *iv_out, size_t iv_len,
+                        uint8_t *mac_out, size_t mac_len,
+                        int is_superencrypted_layer)
+{
+  size_t offset = 0;
+  uint8_t kdf_key[HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN];
+
+  tor_assert(desc);
+  tor_assert(salt);
+  tor_assert(key_out);
+  tor_assert(iv_out);
+  tor_assert(mac_out);
+
+  build_kdf_key(desc, salt, salt_len, kdf_key, sizeof(kdf_key),
+                is_superencrypted_layer);
+  /* Copy the bytes we need for both the secret key and IV. */
+  memcpy(key_out, kdf_key, key_len);
+  offset += key_len;
+  memcpy(iv_out, kdf_key + offset, iv_len);
+  offset += iv_len;
+  memcpy(mac_out, kdf_key + offset, mac_len);
+  /* Extra precaution to make sure we are not out of bound. */
+  tor_assert((offset + mac_len) == sizeof(kdf_key));
+  memwipe(kdf_key, 0, sizeof(kdf_key));
+}
+
+/* === ENCODING === */
+
+/* Encode the given link specifier objects into a newly allocated string.
+ * This can't fail so caller can always assume a valid string being
+ * returned. */
+STATIC char *
+encode_link_specifiers(const smartlist_t *specs)
+{
+  char *encoded_b64 = NULL;
+  link_specifier_list_t *lslist = link_specifier_list_new();
+
+  tor_assert(specs);
+  /* No link specifiers is a code flow error, can't happen. */
+  tor_assert(smartlist_len(specs) > 0);
+  tor_assert(smartlist_len(specs) <= UINT8_MAX);
+
+  link_specifier_list_set_n_spec(lslist, smartlist_len(specs));
+
+  SMARTLIST_FOREACH_BEGIN(specs, const hs_desc_link_specifier_t *,
+                          spec) {
+    link_specifier_t *ls = hs_desc_lspec_to_trunnel(spec);
+    if (ls) {
+      link_specifier_list_add_spec(lslist, ls);
+    }
+  } SMARTLIST_FOREACH_END(spec);
+
+  {
+    uint8_t *encoded;
+    ssize_t encoded_len, encoded_b64_len, ret;
+
+    encoded_len = link_specifier_list_encoded_len(lslist);
+    tor_assert(encoded_len > 0);
+    encoded = tor_malloc_zero(encoded_len);
+    ret = link_specifier_list_encode(encoded, encoded_len, lslist);
+    tor_assert(ret == encoded_len);
+
+    /* Base64 encode our binary format. Add extra NUL byte for the base64
+     * encoded value. */
+    encoded_b64_len = base64_encode_size(encoded_len, 0) + 1;
+    encoded_b64 = tor_malloc_zero(encoded_b64_len);
+    ret = base64_encode(encoded_b64, encoded_b64_len, (const char *) encoded,
+                        encoded_len, 0);
+    tor_assert(ret == (encoded_b64_len - 1));
+    tor_free(encoded);
+  }
+
+  link_specifier_list_free(lslist);
+  return encoded_b64;
+}
+
+/* Encode an introduction point legacy key and certificate. Return a newly
+ * allocated string with it. On failure, return NULL. */
+static char *
+encode_legacy_key(const hs_desc_intro_point_t *ip)
+{
+  char *key_str, b64_cert[256], *encoded = NULL;
+  size_t key_str_len;
+
+  tor_assert(ip);
+
+  /* Encode cross cert. */
+  if (base64_encode(b64_cert, sizeof(b64_cert),
+                    (const char *) ip->legacy.cert.encoded,
+                    ip->legacy.cert.len, BASE64_ENCODE_MULTILINE) < 0) {
+    log_warn(LD_REND, "Unable to encode legacy crosscert.");
+    goto done;
+  }
+  /* Convert the encryption key to PEM format NUL terminated. */
+  if (crypto_pk_write_public_key_to_string(ip->legacy.key, &key_str,
+                                           &key_str_len) < 0) {
+    log_warn(LD_REND, "Unable to encode legacy encryption key.");
+    goto done;
+  }
+  tor_asprintf(&encoded,
+               "%s \n%s"  /* Newline is added by the call above. */
+               "%s\n"
+               "-----BEGIN CROSSCERT-----\n"
+               "%s"
+               "-----END CROSSCERT-----",
+               str_ip_legacy_key, key_str,
+               str_ip_legacy_key_cert, b64_cert);
+  tor_free(key_str);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point encryption key and certificate. Return a newly
+ * allocated string with it. On failure, return NULL. */
+static char *
+encode_enc_key(const hs_desc_intro_point_t *ip)
+{
+  char *encoded = NULL, *encoded_cert;
+  char key_b64[CURVE25519_BASE64_PADDED_LEN + 1];
+
+  tor_assert(ip);
+
+  /* Base64 encode the encryption key for the "enc-key" field. */
+  if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) {
+    goto done;
+  }
+  if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) {
+    goto done;
+  }
+  tor_asprintf(&encoded,
+               "%s ntor %s\n"
+               "%s\n%s",
+               str_ip_enc_key, key_b64,
+               str_ip_enc_key_cert, encoded_cert);
+  tor_free(encoded_cert);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point onion key. Return a newly allocated string
+ * with it. On failure, return NULL. */
+static char *
+encode_onion_key(const hs_desc_intro_point_t *ip)
+{
+  char *encoded = NULL;
+  char key_b64[CURVE25519_BASE64_PADDED_LEN + 1];
+
+  tor_assert(ip);
+
+  /* Base64 encode the encryption key for the "onion-key" field. */
+  if (curve25519_public_to_base64(key_b64, &ip->onion_key) < 0) {
+    goto done;
+  }
+  tor_asprintf(&encoded, "%s ntor %s", str_ip_onion_key, key_b64);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point object and return a newly allocated string
+ * with it. On failure, return NULL. */
+static char *
+encode_intro_point(const ed25519_public_key_t *sig_key,
+                   const hs_desc_intro_point_t *ip)
+{
+  char *encoded_ip = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  tor_assert(ip);
+  tor_assert(sig_key);
+
+  /* Encode link specifier. */
+  {
+    char *ls_str = encode_link_specifiers(ip->link_specifiers);
+    smartlist_add_asprintf(lines, "%s %s", str_intro_point, ls_str);
+    tor_free(ls_str);
+  }
+
+  /* Onion key encoding. */
+  {
+    char *encoded_onion_key = encode_onion_key(ip);
+    if (encoded_onion_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_onion_key);
+    tor_free(encoded_onion_key);
+  }
+
+  /* Authentication key encoding. */
+  {
+    char *encoded_cert;
+    if (tor_cert_encode_ed22519(ip->auth_key_cert, &encoded_cert) < 0) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s\n%s", str_ip_auth_key, encoded_cert);
+    tor_free(encoded_cert);
+  }
+
+  /* Encryption key encoding. */
+  {
+    char *encoded_enc_key = encode_enc_key(ip);
+    if (encoded_enc_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_enc_key);
+    tor_free(encoded_enc_key);
+  }
+
+  /* Legacy key if any. */
+  if (ip->legacy.key != NULL) {
+    /* Strong requirement else the IP creation was badly done. */
+    tor_assert(ip->legacy.cert.encoded);
+    char *encoded_legacy_key = encode_legacy_key(ip);
+    if (encoded_legacy_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_legacy_key);
+    tor_free(encoded_legacy_key);
+  }
+
+  /* Join them all in one blob of text. */
+  encoded_ip = smartlist_join_strings(lines, "\n", 1, NULL);
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+  return encoded_ip;
+}
+
+/* Given a source length, return the new size including padding for the
+ * plaintext encryption. */
+static size_t
+compute_padded_plaintext_length(size_t plaintext_len)
+{
+  size_t plaintext_padded_len;
+  const int padding_block_length = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE;
+
+  /* Make sure we won't overflow. */
+  tor_assert(plaintext_len <= (SIZE_T_CEILING - padding_block_length));
+
+  /* Get the extra length we need to add. For example, if srclen is 10200
+   * bytes, this will expand to (2 * 10k) == 20k thus an extra 9800 bytes. */
+  plaintext_padded_len = CEIL_DIV(plaintext_len, padding_block_length) *
+                         padding_block_length;
+  /* Can never be extra careful. Make sure we are _really_ padded. */
+  tor_assert(!(plaintext_padded_len % padding_block_length));
+  return plaintext_padded_len;
+}
+
+/* Given a buffer, pad it up to the encrypted section padding requirement. Set
+ * the newly allocated string in padded_out and return the length of the
+ * padded buffer. */
+STATIC size_t
+build_plaintext_padding(const char *plaintext, size_t plaintext_len,
+                        uint8_t **padded_out)
+{
+  size_t padded_len;
+  uint8_t *padded;
+
+  tor_assert(plaintext);
+  tor_assert(padded_out);
+
+  /* Allocate the final length including padding. */
+  padded_len = compute_padded_plaintext_length(plaintext_len);
+  tor_assert(padded_len >= plaintext_len);
+  padded = tor_malloc_zero(padded_len);
+
+  memcpy(padded, plaintext, plaintext_len);
+  *padded_out = padded;
+  return padded_len;
+}
+
+/* Using a key, IV and plaintext data of length plaintext_len, create the
+ * encrypted section by encrypting it and setting encrypted_out with the
+ * data. Return size of the encrypted data buffer. */
+static size_t
+build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext,
+                size_t plaintext_len, uint8_t **encrypted_out,
+                int is_superencrypted_layer)
+{
+  size_t encrypted_len;
+  uint8_t *padded_plaintext, *encrypted;
+  crypto_cipher_t *cipher;
+
+  tor_assert(key);
+  tor_assert(iv);
+  tor_assert(plaintext);
+  tor_assert(encrypted_out);
+
+  /* If we are encrypting the middle layer of the descriptor, we need to first
+     pad the plaintext */
+  if (is_superencrypted_layer) {
+    encrypted_len = build_plaintext_padding(plaintext, plaintext_len,
+                                            &padded_plaintext);
+    /* Extra precautions that we have a valid padding length. */
+    tor_assert(!(encrypted_len % HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE));
+  } else { /* No padding required for inner layers */
+    padded_plaintext = tor_memdup(plaintext, plaintext_len);
+    encrypted_len = plaintext_len;
+  }
+
+  /* This creates a cipher for AES. It can't fail. */
+  cipher = crypto_cipher_new_with_iv_and_bits(key, iv,
+                                              HS_DESC_ENCRYPTED_BIT_SIZE);
+  /* We use a stream cipher so the encrypted length will be the same as the
+   * plaintext padded length. */
+  encrypted = tor_malloc_zero(encrypted_len);
+  /* This can't fail. */
+  crypto_cipher_encrypt(cipher, (char *) encrypted,
+                        (const char *) padded_plaintext, encrypted_len);
+  *encrypted_out = encrypted;
+  /* Cleanup. */
+  crypto_cipher_free(cipher);
+  tor_free(padded_plaintext);
+  return encrypted_len;
+}
+
+/* Encrypt the given <b>plaintext</b> buffer using <b>desc</b> to get the
+ * keys. Set encrypted_out with the encrypted data and return the length of
+ * it. <b>is_superencrypted_layer</b> is set if this is the outer encrypted
+ * layer of the descriptor. */
+static size_t
+encrypt_descriptor_data(const hs_descriptor_t *desc, const char *plaintext,
+                        char **encrypted_out, int is_superencrypted_layer)
+{
+  char *final_blob;
+  size_t encrypted_len, final_blob_len, offset = 0;
+  uint8_t *encrypted;
+  uint8_t salt[HS_DESC_ENCRYPTED_SALT_LEN];
+  uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN];
+  uint8_t mac_key[DIGEST256_LEN], mac[DIGEST256_LEN];
+
+  tor_assert(desc);
+  tor_assert(plaintext);
+  tor_assert(encrypted_out);
+
+  /* Get our salt. The returned bytes are already hashed. */
+  crypto_strongest_rand(salt, sizeof(salt));
+
+  /* KDF construction resulting in a key from which the secret key, IV and MAC
+   * key are extracted which is what we need for the encryption. */
+  build_secret_key_iv_mac(desc, salt, sizeof(salt),
+                          secret_key, sizeof(secret_key),
+                          secret_iv, sizeof(secret_iv),
+                          mac_key, sizeof(mac_key),
+                          is_superencrypted_layer);
+
+  /* Build the encrypted part that is do the actual encryption. */
+  encrypted_len = build_encrypted(secret_key, secret_iv, plaintext,
+                                  strlen(plaintext), &encrypted,
+                                  is_superencrypted_layer);
+  memwipe(secret_key, 0, sizeof(secret_key));
+  memwipe(secret_iv, 0, sizeof(secret_iv));
+  /* This construction is specified in section 2.5 of proposal 224. */
+  final_blob_len = sizeof(salt) + encrypted_len + DIGEST256_LEN;
+  final_blob = tor_malloc_zero(final_blob_len);
+
+  /* Build the MAC. */
+  build_mac(mac_key, sizeof(mac_key), salt, sizeof(salt),
+            encrypted, encrypted_len, mac, sizeof(mac));
+  memwipe(mac_key, 0, sizeof(mac_key));
+
+  /* The salt is the first value. */
+  memcpy(final_blob, salt, sizeof(salt));
+  offset = sizeof(salt);
+  /* Second value is the encrypted data. */
+  memcpy(final_blob + offset, encrypted, encrypted_len);
+  offset += encrypted_len;
+  /* Third value is the MAC. */
+  memcpy(final_blob + offset, mac, sizeof(mac));
+  offset += sizeof(mac);
+  /* Cleanup the buffers. */
+  memwipe(salt, 0, sizeof(salt));
+  memwipe(encrypted, 0, encrypted_len);
+  tor_free(encrypted);
+  /* Extra precaution. */
+  tor_assert(offset == final_blob_len);
+
+  *encrypted_out = final_blob;
+  return final_blob_len;
+}
+
+/* Create and return a string containing a fake client-auth entry. It's the
+ * responsibility of the caller to free the returned string. This function will
+ * never fail. */
+static char *
+get_fake_auth_client_str(void)
+{
+  char *auth_client_str = NULL;
+  /* We are gonna fill these arrays with fake base64 data. They are all double
+   * the size of their binary representation to fit the base64 overhead. */
+  char client_id_b64[8*2];
+  char iv_b64[16*2];
+  char encrypted_cookie_b64[16*2];
+  int retval;
+
+  /* This is a macro to fill a field with random data and then base64 it. */
+#define FILL_WITH_FAKE_DATA_AND_BASE64(field) STMT_BEGIN         \
+  crypto_rand((char *)field, sizeof(field));                     \
+  retval = base64_encode_nopad(field##_b64, sizeof(field##_b64), \
+                               field, sizeof(field));            \
+  tor_assert(retval > 0);                                        \
+  STMT_END
+
+  { /* Get those fakes! */
+    uint8_t client_id[8]; /* fake client-id */
+    uint8_t iv[16]; /* fake IV (initialization vector) */
+    uint8_t encrypted_cookie[16]; /* fake encrypted cookie */
+
+    FILL_WITH_FAKE_DATA_AND_BASE64(client_id);
+    FILL_WITH_FAKE_DATA_AND_BASE64(iv);
+    FILL_WITH_FAKE_DATA_AND_BASE64(encrypted_cookie);
+  }
+
+  /* Build the final string */
+  tor_asprintf(&auth_client_str, "%s %s %s %s", str_desc_auth_client,
+               client_id_b64, iv_b64, encrypted_cookie_b64);
+
+#undef FILL_WITH_FAKE_DATA_AND_BASE64
+
+  return auth_client_str;
+}
+
+/** How many lines of "client-auth" we want in our descriptors; fake or not. */
+#define CLIENT_AUTH_ENTRIES_BLOCK_SIZE 16
+
+/** Create the "client-auth" part of the descriptor and return a
+ *  newly-allocated string with it. It's the responsibility of the caller to
+ *  free the returned string. */
+static char *
+get_fake_auth_client_lines(void)
+{
+  /* XXX: Client authorization is still not implemented, so all this function
+     does is make fake clients */
+  int i = 0;
+  smartlist_t *auth_client_lines = smartlist_new();
+  char *auth_client_lines_str = NULL;
+
+  /* Make a line for each fake client */
+  const int num_fake_clients = CLIENT_AUTH_ENTRIES_BLOCK_SIZE;
+  for (i = 0; i < num_fake_clients; i++) {
+    char *auth_client_str = get_fake_auth_client_str();
+    tor_assert(auth_client_str);
+    smartlist_add(auth_client_lines, auth_client_str);
+  }
+
+  /* Join all lines together to form final string */
+  auth_client_lines_str = smartlist_join_strings(auth_client_lines,
+                                                 "\n", 1, NULL);
+  /* Cleanup the mess */
+  SMARTLIST_FOREACH(auth_client_lines, char *, a, tor_free(a));
+  smartlist_free(auth_client_lines);
+
+  return auth_client_lines_str;
+}
+
+/* Create the inner layer of the descriptor (which includes the intro points,
+ * etc.). Return a newly-allocated string with the layer plaintext, or NULL if
+ * an error occurred. It's the responsibility of the caller to free the
+ * returned string. */
+static char *
+get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc)
+{
+  char *encoded_str = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  /* Build the start of the section prior to the introduction points. */
+  {
+    if (!desc->encrypted_data.create2_ntor) {
+      log_err(LD_BUG, "HS desc doesn't have recognized handshake type.");
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s %d\n", str_create2_formats,
+                           ONION_HANDSHAKE_TYPE_NTOR);
+
+    if (desc->encrypted_data.intro_auth_types &&
+        smartlist_len(desc->encrypted_data.intro_auth_types)) {
+      /* Put the authentication-required line. */
+      char *buf = smartlist_join_strings(desc->encrypted_data.intro_auth_types,
+                                         " ", 0, NULL);
+      smartlist_add_asprintf(lines, "%s %s\n", str_intro_auth_required, buf);
+      tor_free(buf);
+    }
+
+    if (desc->encrypted_data.single_onion_service) {
+      smartlist_add_asprintf(lines, "%s\n", str_single_onion);
+    }
+  }
+
+  /* Build the introduction point(s) section. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    char *encoded_ip = encode_intro_point(&desc->plaintext_data.signing_pubkey,
+                                          ip);
+    if (encoded_ip == NULL) {
+      log_err(LD_BUG, "HS desc intro point is malformed.");
+      goto err;
+    }
+    smartlist_add(lines, encoded_ip);
+  } SMARTLIST_FOREACH_END(ip);
+
+  /* Build the entire encrypted data section into one encoded plaintext and
+   * then encrypt it. */
+  encoded_str = smartlist_join_strings(lines, "", 0, NULL);
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+
+  return encoded_str;
+}
+
+/* Create the middle layer of the descriptor, which includes the client auth
+ * data and the encrypted inner layer (provided as a base64 string at
+ * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the
+ * layer plaintext, or NULL if an error occurred. It's the responsibility of
+ * the caller to free the returned string. */
+static char *
+get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
+                                    const char *layer2_b64_ciphertext)
+{
+  char *layer1_str = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  /* XXX: Disclaimer: This function generates only _fake_ client auth
+   * data. Real client auth is not yet implemented, but client auth data MUST
+   * always be present in descriptors. In the future this function will be
+   * refactored to use real client auth data if they exist (#20700). */
+  (void) *desc;
+
+  /* Specify auth type */
+  smartlist_add_asprintf(lines, "%s %s\n", str_desc_auth_type, "x25519");
+
+  {  /* Create fake ephemeral x25519 key */
+    char fake_key_base64[CURVE25519_BASE64_PADDED_LEN + 1];
+    curve25519_keypair_t fake_x25519_keypair;
+    if (curve25519_keypair_generate(&fake_x25519_keypair, 0) < 0) {
+      goto done;
+    }
+    if (curve25519_public_to_base64(fake_key_base64,
+                                    &fake_x25519_keypair.pubkey) < 0) {
+      goto done;
+    }
+    smartlist_add_asprintf(lines, "%s %s\n",
+                           str_desc_auth_key, fake_key_base64);
+    /* No need to memwipe any of these fake keys. They will go unused. */
+  }
+
+  {  /* Create fake auth-client lines. */
+    char *auth_client_lines = get_fake_auth_client_lines();
+    tor_assert(auth_client_lines);
+    smartlist_add(lines, auth_client_lines);
+  }
+
+  /* create encrypted section */
+  {
+    smartlist_add_asprintf(lines,
+                           "%s\n"
+                           "-----BEGIN MESSAGE-----\n"
+                           "%s"
+                           "-----END MESSAGE-----",
+                           str_encrypted, layer2_b64_ciphertext);
+  }
+
+  layer1_str = smartlist_join_strings(lines, "", 0, NULL);
+
+ done:
+  SMARTLIST_FOREACH(lines, char *, a, tor_free(a));
+  smartlist_free(lines);
+
+  return layer1_str;
+}
+
+/* Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before
+ * returning it. <b>desc</b> is provided to derive the encryption
+ * keys. <b>is_superencrypted_layer</b> is set if <b>encoded_str</b> is the
+ * middle (superencrypted) layer of the descriptor. It's the responsibility of
+ * the caller to free the returned string. */
+static char *
+encrypt_desc_data_and_base64(const hs_descriptor_t *desc,
+                             const char *encoded_str,
+                             int is_superencrypted_layer)
+{
+  char *enc_b64;
+  ssize_t enc_b64_len, ret_len, enc_len;
+  char *encrypted_blob = NULL;
+
+  enc_len = encrypt_descriptor_data(desc, encoded_str, &encrypted_blob,
+                                    is_superencrypted_layer);
+  /* Get the encoded size plus a NUL terminating byte. */
+  enc_b64_len = base64_encode_size(enc_len, BASE64_ENCODE_MULTILINE) + 1;
+  enc_b64 = tor_malloc_zero(enc_b64_len);
+  /* Base64 the encrypted blob before returning it. */
+  ret_len = base64_encode(enc_b64, enc_b64_len, encrypted_blob, enc_len,
+                          BASE64_ENCODE_MULTILINE);
+  /* Return length doesn't count the NUL byte. */
+  tor_assert(ret_len == (enc_b64_len - 1));
+  tor_free(encrypted_blob);
+
+  return enc_b64;
+}
+
+/* Generate and encode the superencrypted portion of <b>desc</b>. This also
+ * involves generating the encrypted portion of the descriptor, and performing
+ * the superencryption. A newly allocated NUL-terminated string pointer
+ * containing the encrypted encoded blob is put in encrypted_blob_out. Return 0
+ * on success else a negative value. */
+static int
+encode_superencrypted_data(const hs_descriptor_t *desc,
+                           char **encrypted_blob_out)
+{
+  int ret = -1;
+  char *layer2_str = NULL;
+  char *layer2_b64_ciphertext = NULL;
+  char *layer1_str = NULL;
+  char *layer1_b64_ciphertext = NULL;
+
+  tor_assert(desc);
+  tor_assert(encrypted_blob_out);
+
+  /* Func logic: We first create the inner layer of the descriptor (layer2).
+   * We then encrypt it and use it to create the middle layer of the descriptor
+   * (layer1).  Finally we superencrypt the middle layer and return it to our
+   * caller. */
+
+  /* Create inner descriptor layer */
+  layer2_str = get_inner_encrypted_layer_plaintext(desc);
+  if (!layer2_str) {
+    goto err;
+  }
+
+  /* Encrypt and b64 the inner layer */
+  layer2_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer2_str, 0);
+  if (!layer2_b64_ciphertext) {
+    goto err;
+  }
+
+  /* Now create middle descriptor layer given the inner layer */
+  layer1_str = get_outer_encrypted_layer_plaintext(desc,layer2_b64_ciphertext);
+  if (!layer1_str) {
+    goto err;
+  }
+
+  /* Encrypt and base64 the middle layer */
+  layer1_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer1_str, 1);
+  if (!layer1_b64_ciphertext) {
+    goto err;
+  }
+
+  /* Success! */
+  ret = 0;
+
+ err:
+  tor_free(layer1_str);
+  tor_free(layer2_str);
+  tor_free(layer2_b64_ciphertext);
+
+  *encrypted_blob_out = layer1_b64_ciphertext;
+  return ret;
+}
+
+/* Encode a v3 HS descriptor. Return 0 on success and set encoded_out to the
+ * newly allocated string of the encoded descriptor. On error, -1 is returned
+ * and encoded_out is untouched. */
+static int
+desc_encode_v3(const hs_descriptor_t *desc,
+               const ed25519_keypair_t *signing_kp, char **encoded_out)
+{
+  int ret = -1;
+  char *encoded_str = NULL;
+  size_t encoded_len;
+  smartlist_t *lines = smartlist_new();
+
+  tor_assert(desc);
+  tor_assert(signing_kp);
+  tor_assert(encoded_out);
+  tor_assert(desc->plaintext_data.version == 3);
+
+  if (BUG(desc->subcredential == NULL)) {
+    goto err;
+  }
+
+  /* Build the non-encrypted values. */
+  {
+    char *encoded_cert;
+    /* Encode certificate then create the first line of the descriptor. */
+    if (desc->plaintext_data.signing_key_cert->cert_type
+        != CERT_TYPE_SIGNING_HS_DESC) {
+      log_err(LD_BUG, "HS descriptor signing key has an unexpected cert type "
+              "(%d)", (int) desc->plaintext_data.signing_key_cert->cert_type);
+      goto err;
+    }
+    if (tor_cert_encode_ed22519(desc->plaintext_data.signing_key_cert,
+                                &encoded_cert) < 0) {
+      /* The function will print error logs. */
+      goto err;
+    }
+    /* Create the hs descriptor line. */
+    smartlist_add_asprintf(lines, "%s %" PRIu32, str_hs_desc,
+                           desc->plaintext_data.version);
+    /* Add the descriptor lifetime line (in minutes). */
+    smartlist_add_asprintf(lines, "%s %" PRIu32, str_lifetime,
+                           desc->plaintext_data.lifetime_sec / 60);
+    /* Create the descriptor certificate line. */
+    smartlist_add_asprintf(lines, "%s\n%s", str_desc_cert, encoded_cert);
+    tor_free(encoded_cert);
+    /* Create the revision counter line. */
+    smartlist_add_asprintf(lines, "%s %" PRIu64, str_rev_counter,
+                           desc->plaintext_data.revision_counter);
+  }
+
+  /* Build the superencrypted data section. */
+  {
+    char *enc_b64_blob=NULL;
+    if (encode_superencrypted_data(desc, &enc_b64_blob) < 0) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines,
+                           "%s\n"
+                           "-----BEGIN MESSAGE-----\n"
+                           "%s"
+                           "-----END MESSAGE-----",
+                           str_superencrypted, enc_b64_blob);
+    tor_free(enc_b64_blob);
+  }
+
+  /* Join all lines in one string so we can generate a signature and append
+   * it to the descriptor. */
+  encoded_str = smartlist_join_strings(lines, "\n", 1, &encoded_len);
+
+  /* Sign all fields of the descriptor with our short term signing key. */
+  {
+    ed25519_signature_t sig;
+    char ed_sig_b64[ED25519_SIG_BASE64_LEN + 1];
+    if (ed25519_sign_prefixed(&sig,
+                              (const uint8_t *) encoded_str, encoded_len,
+                              str_desc_sig_prefix, signing_kp) < 0) {
+      log_warn(LD_BUG, "Can't sign encoded HS descriptor!");
+      tor_free(encoded_str);
+      goto err;
+    }
+    if (ed25519_signature_to_base64(ed_sig_b64, &sig) < 0) {
+      log_warn(LD_BUG, "Can't base64 encode descriptor signature!");
+      tor_free(encoded_str);
+      goto err;
+    }
+    /* Create the signature line. */
+    smartlist_add_asprintf(lines, "%s %s", str_signature, ed_sig_b64);
+  }
+  /* Free previous string that we used so compute the signature. */
+  tor_free(encoded_str);
+  encoded_str = smartlist_join_strings(lines, "\n", 1, NULL);
+  *encoded_out = encoded_str;
+
+  if (strlen(encoded_str) >= hs_cache_get_max_descriptor_size()) {
+    log_warn(LD_GENERAL, "We just made an HS descriptor that's too big (%d)."
+             "Failing.", (int)strlen(encoded_str));
+    tor_free(encoded_str);
+    goto err;
+  }
+
+  /* XXX: Trigger a control port event. */
+
+  /* Success! */
+  ret = 0;
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+  return ret;
+}
+
+/* === DECODING === */
+
+/* Given an encoded string of the link specifiers, return a newly allocated
+ * list of decoded link specifiers. Return NULL on error. */
+STATIC smartlist_t *
+decode_link_specifiers(const char *encoded)
+{
+  int decoded_len;
+  size_t encoded_len, i;
+  uint8_t *decoded;
+  smartlist_t *results = NULL;
+  link_specifier_list_t *specs = NULL;
+
+  tor_assert(encoded);
+
+  encoded_len = strlen(encoded);
+  decoded = tor_malloc(encoded_len);
+  decoded_len = base64_decode((char *) decoded, encoded_len, encoded,
+                              encoded_len);
+  if (decoded_len < 0) {
+    goto err;
+  }
+
+  if (link_specifier_list_parse(&specs, decoded,
+                                (size_t) decoded_len) < decoded_len) {
+    goto err;
+  }
+  tor_assert(specs);
+  results = smartlist_new();
+
+  for (i = 0; i < link_specifier_list_getlen_spec(specs); i++) {
+    hs_desc_link_specifier_t *hs_spec;
+    link_specifier_t *ls = link_specifier_list_get_spec(specs, i);
+    tor_assert(ls);
+
+    hs_spec = tor_malloc_zero(sizeof(*hs_spec));
+    hs_spec->type = link_specifier_get_ls_type(ls);
+    switch (hs_spec->type) {
+    case LS_IPV4:
+      tor_addr_from_ipv4h(&hs_spec->u.ap.addr,
+                          link_specifier_get_un_ipv4_addr(ls));
+      hs_spec->u.ap.port = link_specifier_get_un_ipv4_port(ls);
+      break;
+    case LS_IPV6:
+      tor_addr_from_ipv6_bytes(&hs_spec->u.ap.addr, (const char *)
+                               link_specifier_getarray_un_ipv6_addr(ls));
+      hs_spec->u.ap.port = link_specifier_get_un_ipv6_port(ls);
+      break;
+    case LS_LEGACY_ID:
+      /* Both are known at compile time so let's make sure they are the same
+       * else we can copy memory out of bound. */
+      tor_assert(link_specifier_getlen_un_legacy_id(ls) ==
+                 sizeof(hs_spec->u.legacy_id));
+      memcpy(hs_spec->u.legacy_id, link_specifier_getarray_un_legacy_id(ls),
+             sizeof(hs_spec->u.legacy_id));
+      break;
+    case LS_ED25519_ID:
+      /* Both are known at compile time so let's make sure they are the same
+       * else we can copy memory out of bound. */
+      tor_assert(link_specifier_getlen_un_ed25519_id(ls) ==
+                 sizeof(hs_spec->u.ed25519_id));
+      memcpy(hs_spec->u.ed25519_id,
+             link_specifier_getconstarray_un_ed25519_id(ls),
+             sizeof(hs_spec->u.ed25519_id));
+      break;
+    default:
+      tor_free(hs_spec);
+      goto err;
+    }
+
+    smartlist_add(results, hs_spec);
+  }
+
+  goto done;
+ err:
+  if (results) {
+    SMARTLIST_FOREACH(results, hs_desc_link_specifier_t *, s, tor_free(s));
+    smartlist_free(results);
+    results = NULL;
+  }
+ done:
+  link_specifier_list_free(specs);
+  tor_free(decoded);
+  return results;
+}
+
+/* Given a list of authentication types, decode it and put it in the encrypted
+ * data section. Return 1 if we at least know one of the type or 0 if we know
+ * none of them. */
+static int
+decode_auth_type(hs_desc_encrypted_data_t *desc, const char *list)
+{
+  int match = 0;
+
+  tor_assert(desc);
+  tor_assert(list);
+
+  desc->intro_auth_types = smartlist_new();
+  smartlist_split_string(desc->intro_auth_types, list, " ", 0, 0);
+
+  /* Validate the types that we at least know about one. */
+  SMARTLIST_FOREACH_BEGIN(desc->intro_auth_types, const char *, auth) {
+    for (int idx = 0; intro_auth_types[idx].identifier; idx++) {
+      if (!strncmp(auth, intro_auth_types[idx].identifier,
+                   strlen(intro_auth_types[idx].identifier))) {
+        match = 1;
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(auth);
+
+  return match;
+}
+
+/* Parse a space-delimited list of integers representing CREATE2 formats into
+ * the bitfield in hs_desc_encrypted_data_t. Ignore unrecognized values. */
+static void
+decode_create2_list(hs_desc_encrypted_data_t *desc, const char *list)
+{
+  smartlist_t *tokens;
+
+  tor_assert(desc);
+  tor_assert(list);
+
+  tokens = smartlist_new();
+  smartlist_split_string(tokens, list, " ", 0, 0);
+
+  SMARTLIST_FOREACH_BEGIN(tokens, char *, s) {
+    int ok;
+    unsigned long type = tor_parse_ulong(s, 10, 1, UINT16_MAX, &ok, NULL);
+    if (!ok) {
+      log_warn(LD_REND, "Unparseable value %s in create2 list", escaped(s));
+      continue;
+    }
+    switch (type) {
+    case ONION_HANDSHAKE_TYPE_NTOR:
+      desc->create2_ntor = 1;
+      break;
+    default:
+      /* We deliberately ignore unsupported handshake types */
+      continue;
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+  SMARTLIST_FOREACH(tokens, char *, s, tor_free(s));
+  smartlist_free(tokens);
+}
+
+/* Given a certificate, validate the certificate for certain conditions which
+ * are if the given type matches the cert's one, if the signing key is
+ * included and if the that key was actually used to sign the certificate.
+ *
+ * Return 1 iff if all conditions pass or 0 if one of them fails. */
+STATIC int
+cert_is_valid(tor_cert_t *cert, uint8_t type, const char *log_obj_type)
+{
+  tor_assert(log_obj_type);
+
+  if (cert == NULL) {
+    log_warn(LD_REND, "Certificate for %s couldn't be parsed.", log_obj_type);
+    goto err;
+  }
+  if (cert->cert_type != type) {
+    log_warn(LD_REND, "Invalid cert type %02x for %s.", cert->cert_type,
+             log_obj_type);
+    goto err;
+  }
+  /* All certificate must have its signing key included. */
+  if (!cert->signing_key_included) {
+    log_warn(LD_REND, "Signing key is NOT included for %s.", log_obj_type);
+    goto err;
+  }
+  /* The following will not only check if the signature matches but also the
+   * expiration date and overall validity. */
+  if (tor_cert_checksig(cert, &cert->signing_key, approx_time()) < 0) {
+    log_warn(LD_REND, "Invalid signature for %s: %s", log_obj_type,
+             tor_cert_describe_signature_status(cert));
+    goto err;
+  }
+
+  return 1;
+ err:
+  return 0;
+}
+
+/* Given some binary data, try to parse it to get a certificate object. If we
+ * have a valid cert, validate it using the given wanted type. On error, print
+ * a log using the err_msg has the certificate identifier adding semantic to
+ * the log and cert_out is set to NULL. On success, 0 is returned and cert_out
+ * points to a newly allocated certificate object. */
+static int
+cert_parse_and_validate(tor_cert_t **cert_out, const char *data,
+                        size_t data_len, unsigned int cert_type_wanted,
+                        const char *err_msg)
+{
+  tor_cert_t *cert;
+
+  tor_assert(cert_out);
+  tor_assert(data);
+  tor_assert(err_msg);
+
+  /* Parse certificate. */
+  cert = tor_cert_parse((const uint8_t *) data, data_len);
+  if (!cert) {
+    log_warn(LD_REND, "Certificate for %s couldn't be parsed.", err_msg);
+    goto err;
+  }
+
+  /* Validate certificate. */
+  if (!cert_is_valid(cert, cert_type_wanted, err_msg)) {
+    goto err;
+  }
+
+  *cert_out = cert;
+  return 0;
+
+ err:
+  tor_cert_free(cert);
+  *cert_out = NULL;
+  return -1;
+}
+
+/* Return true iff the given length of the encrypted data of a descriptor
+ * passes validation. */
+STATIC int
+encrypted_data_length_is_valid(size_t len)
+{
+  /* Make sure there is enough data for the salt and the mac. The equality is
+     there to ensure that there is at least one byte of encrypted data. */
+  if (len <= HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN) {
+    log_warn(LD_REND, "Length of descriptor's encrypted data is too small. "
+                      "Got %lu but minimum value is %d",
+             (unsigned long)len, HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN);
+    goto err;
+  }
+
+  return 1;
+ err:
+  return 0;
+}
+
+/** Decrypt an encrypted descriptor layer at <b>encrypted_blob</b> of size
+ *  <b>encrypted_blob_size</b>. Use the descriptor object <b>desc</b> to
+ *  generate the right decryption keys; set <b>decrypted_out</b> to the
+ *  plaintext. If <b>is_superencrypted_layer</b> is set, this is the outter
+ *  encrypted layer of the descriptor.
+ *
+ * On any error case, including an empty output, return 0 and set
+ * *<b>decrypted_out</b> to NULL.
+ */
+MOCK_IMPL(STATIC size_t,
+decrypt_desc_layer,(const hs_descriptor_t *desc,
+                    const uint8_t *encrypted_blob,
+                    size_t encrypted_blob_size,
+                    int is_superencrypted_layer,
+                    char **decrypted_out))
+{
+  uint8_t *decrypted = NULL;
+  uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN];
+  uint8_t mac_key[DIGEST256_LEN], our_mac[DIGEST256_LEN];
+  const uint8_t *salt, *encrypted, *desc_mac;
+  size_t encrypted_len, result_len = 0;
+
+  tor_assert(decrypted_out);
+  tor_assert(desc);
+  tor_assert(encrypted_blob);
+
+  /* Construction is as follow: SALT | ENCRYPTED_DATA | MAC .
+   * Make sure we have enough space for all these things. */
+  if (!encrypted_data_length_is_valid(encrypted_blob_size)) {
+    goto err;
+  }
+
+  /* Start of the blob thus the salt. */
+  salt = encrypted_blob;
+
+  /* Next is the encrypted data. */
+  encrypted = encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN;
+  encrypted_len = encrypted_blob_size -
+    (HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN);
+  tor_assert(encrypted_len > 0); /* guaranteed by the check above */
+
+  /* And last comes the MAC. */
+  desc_mac = encrypted_blob + encrypted_blob_size - DIGEST256_LEN;
+
+  /* KDF construction resulting in a key from which the secret key, IV and MAC
+   * key are extracted which is what we need for the decryption. */
+  build_secret_key_iv_mac(desc, salt, HS_DESC_ENCRYPTED_SALT_LEN,
+                          secret_key, sizeof(secret_key),
+                          secret_iv, sizeof(secret_iv),
+                          mac_key, sizeof(mac_key),
+                          is_superencrypted_layer);
+
+  /* Build MAC. */
+  build_mac(mac_key, sizeof(mac_key), salt, HS_DESC_ENCRYPTED_SALT_LEN,
+            encrypted, encrypted_len, our_mac, sizeof(our_mac));
+  memwipe(mac_key, 0, sizeof(mac_key));
+  /* Verify MAC; MAC is H(mac_key || salt || encrypted)
+   *
+   * This is a critical check that is making sure the computed MAC matches the
+   * one in the descriptor. */
+  if (!tor_memeq(our_mac, desc_mac, sizeof(our_mac))) {
+    log_warn(LD_REND, "Encrypted service descriptor MAC check failed");
+    goto err;
+  }
+
+  {
+    /* Decrypt. Here we are assured that the encrypted length is valid for
+     * decryption. */
+    crypto_cipher_t *cipher;
+
+    cipher = crypto_cipher_new_with_iv_and_bits(secret_key, secret_iv,
+                                                HS_DESC_ENCRYPTED_BIT_SIZE);
+    /* Extra byte for the NUL terminated byte. */
+    decrypted = tor_malloc_zero(encrypted_len + 1);
+    crypto_cipher_decrypt(cipher, (char *) decrypted,
+                          (const char *) encrypted, encrypted_len);
+    crypto_cipher_free(cipher);
+  }
+
+  {
+    /* Adjust length to remove NUL padding bytes */
+    uint8_t *end = memchr(decrypted, 0, encrypted_len);
+    result_len = encrypted_len;
+    if (end) {
+      result_len = end - decrypted;
+    }
+  }
+
+  if (result_len == 0) {
+    /* Treat this as an error, so that somebody will free the output. */
+    goto err;
+  }
+
+  /* Make sure to NUL terminate the string. */
+  decrypted[encrypted_len] = '\0';
+  *decrypted_out = (char *) decrypted;
+  goto done;
+
+ err:
+  if (decrypted) {
+    tor_free(decrypted);
+  }
+  *decrypted_out = NULL;
+  result_len = 0;
+
+ done:
+  memwipe(secret_key, 0, sizeof(secret_key));
+  memwipe(secret_iv, 0, sizeof(secret_iv));
+  return result_len;
+}
+
+/* Basic validation that the superencrypted client auth portion of the
+ * descriptor is well-formed and recognized. Return True if so, otherwise
+ * return False. */
+static int
+superencrypted_auth_data_is_valid(smartlist_t *tokens)
+{
+  /* XXX: This is just basic validation for now. When we implement client auth,
+     we can refactor this function so that it actually parses and saves the
+     data. */
+
+  { /* verify desc auth type */
+    const directory_token_t *tok;
+    tok = find_by_keyword(tokens, R3_DESC_AUTH_TYPE);
+    tor_assert(tok->n_args >= 1);
+    if (strcmp(tok->args[0], "x25519")) {
+      log_warn(LD_DIR, "Unrecognized desc auth type");
+      return 0;
+    }
+  }
+
+  { /* verify desc auth key */
+    const directory_token_t *tok;
+    curve25519_public_key_t k;
+    tok = find_by_keyword(tokens, R3_DESC_AUTH_KEY);
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus desc auth key in HS desc");
+      return 0;
+    }
+  }
+
+  /* verify desc auth client items */
+  SMARTLIST_FOREACH_BEGIN(tokens, const directory_token_t *, tok) {
+    if (tok->tp == R3_DESC_AUTH_CLIENT) {
+      tor_assert(tok->n_args >= 3);
+    }
+  } SMARTLIST_FOREACH_END(tok);
+
+  return 1;
+}
+
+/* Parse <b>message</b>, the plaintext of the superencrypted portion of an HS
+ * descriptor. Set <b>encrypted_out</b> to the encrypted blob, and return its
+ * size */
+STATIC size_t
+decode_superencrypted(const char *message, size_t message_len,
+                     uint8_t **encrypted_out)
+{
+  int retval = 0;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, message, message + message_len, tokens,
+                      hs_desc_superencrypted_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Superencrypted portion is not parseable");
+    goto err;
+  }
+
+  /* Do some rudimentary validation of the authentication data */
+  if (!superencrypted_auth_data_is_valid(tokens)) {
+    log_warn(LD_REND, "Invalid auth data");
+    goto err;
+  }
+
+  /* Extract the encrypted data section. */
+  {
+    const directory_token_t *tok;
+    tok = find_by_keyword(tokens, R3_ENCRYPTED);
+    tor_assert(tok->object_body);
+    if (strcmp(tok->object_type, "MESSAGE") != 0) {
+      log_warn(LD_REND, "Desc superencrypted data section is invalid");
+      goto err;
+    }
+    /* Make sure the length of the encrypted blob is valid. */
+    if (!encrypted_data_length_is_valid(tok->object_size)) {
+      goto err;
+    }
+
+    /* Copy the encrypted blob to the descriptor object so we can handle it
+     * latter if needed. */
+    tor_assert(tok->object_size <= INT_MAX);
+    *encrypted_out = tor_memdup(tok->object_body, tok->object_size);
+    retval = (int) tok->object_size;
+  }
+
+ err:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    memarea_drop_all(area);
+  }
+
+  return retval;
+}
+
+/* Decrypt both the superencrypted and the encrypted section of the descriptor
+ * using the given descriptor object <b>desc</b>. A newly allocated NUL
+ * terminated string is put in decrypted_out which contains the inner encrypted
+ * layer of the descriptor. Return the length of decrypted_out on success else
+ * 0 is returned and decrypted_out is set to NULL. */
+static size_t
+desc_decrypt_all(const hs_descriptor_t *desc, char **decrypted_out)
+{
+  size_t  decrypted_len = 0;
+  size_t encrypted_len = 0;
+  size_t superencrypted_len = 0;
+  char *superencrypted_plaintext = NULL;
+  uint8_t *encrypted_blob = NULL;
+
+  /** Function logic: This function takes us from the descriptor header to the
+   *  inner encrypted layer, by decrypting and decoding the middle descriptor
+   *  layer. In the end we return the contents of the inner encrypted layer to
+   *  our caller. */
+
+  /* 1. Decrypt middle layer of descriptor */
+  superencrypted_len = decrypt_desc_layer(desc,
+                                 desc->plaintext_data.superencrypted_blob,
+                                 desc->plaintext_data.superencrypted_blob_size,
+                                 1,
+                                 &superencrypted_plaintext);
+  if (!superencrypted_len) {
+    log_warn(LD_REND, "Decrypting superencrypted desc failed.");
+    goto err;
+  }
+  tor_assert(superencrypted_plaintext);
+
+  /* 2. Parse "superencrypted" */
+  encrypted_len = decode_superencrypted(superencrypted_plaintext,
+                                        superencrypted_len,
+                                        &encrypted_blob);
+  if (!encrypted_len) {
+    log_warn(LD_REND, "Decrypting encrypted desc failed.");
+    goto err;
+  }
+  tor_assert(encrypted_blob);
+
+  /* 3. Decrypt "encrypted" and set decrypted_out */
+  char *decrypted_desc;
+  decrypted_len = decrypt_desc_layer(desc,
+                                     encrypted_blob, encrypted_len,
+                                     0, &decrypted_desc);
+  if (!decrypted_len) {
+    log_warn(LD_REND, "Decrypting encrypted desc failed.");
+    goto err;
+  }
+  tor_assert(decrypted_desc);
+
+  *decrypted_out = decrypted_desc;
+
+ err:
+  tor_free(superencrypted_plaintext);
+  tor_free(encrypted_blob);
+
+  return decrypted_len;
+}
+
+/* Given the token tok for an intro point legacy key, the list of tokens, the
+ * introduction point ip being decoded and the descriptor desc from which it
+ * comes from, decode the legacy key and set the intro point object. Return 0
+ * on success else -1 on failure. */
+static int
+decode_intro_legacy_key(const directory_token_t *tok,
+                        smartlist_t *tokens,
+                        hs_desc_intro_point_t *ip,
+                        const hs_descriptor_t *desc)
+{
+  tor_assert(tok);
+  tor_assert(tokens);
+  tor_assert(ip);
+  tor_assert(desc);
+
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_REND, "Introduction point legacy key is invalid");
+    goto err;
+  }
+  ip->legacy.key = crypto_pk_dup_key(tok->key);
+  /* Extract the legacy cross certification cert which MUST be present if we
+   * have a legacy key. */
+  tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY_CERT);
+  if (!tok) {
+    log_warn(LD_REND, "Introduction point legacy key cert is missing");
+    goto err;
+  }
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "CROSSCERT")) {
+    /* Info level because this might be an unknown field that we should
+     * ignore. */
+    log_info(LD_REND, "Introduction point legacy encryption key "
+                      "cross-certification has an unknown format.");
+    goto err;
+  }
+  /* Keep a copy of the certificate. */
+  ip->legacy.cert.encoded = tor_memdup(tok->object_body, tok->object_size);
+  ip->legacy.cert.len = tok->object_size;
+  /* The check on the expiration date is for the entire lifetime of a
+   * certificate which is 24 hours. However, a descriptor has a maximum
+   * lifetime of 12 hours meaning we have a 12h difference between the two
+   * which ultimately accommodate the clock skewed client. */
+  if (rsa_ed25519_crosscert_check(ip->legacy.cert.encoded,
+                                  ip->legacy.cert.len, ip->legacy.key,
+                                  &desc->plaintext_data.signing_pubkey,
+                                  approx_time() - HS_DESC_CERT_LIFETIME)) {
+    log_warn(LD_REND, "Unable to check cross-certification on the "
+                      "introduction point legacy encryption key.");
+    ip->cross_certified = 0;
+    goto err;
+  }
+
+  /* Success. */
+  return 0;
+ err:
+  return -1;
+}
+
+/* Dig into the descriptor <b>tokens</b> to find the onion key we should use
+ * for this intro point, and set it into <b>onion_key_out</b>. Return 0 if it
+ * was found and well-formed, otherwise return -1 in case of errors. */
+static int
+set_intro_point_onion_key(curve25519_public_key_t *onion_key_out,
+                          const smartlist_t *tokens)
+{
+  int retval = -1;
+  smartlist_t *onion_keys = NULL;
+
+  tor_assert(onion_key_out);
+
+  onion_keys = find_all_by_keyword(tokens, R3_INTRO_ONION_KEY);
+  if (!onion_keys) {
+    log_warn(LD_REND, "Descriptor did not contain intro onion keys");
+    goto err;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(onion_keys, directory_token_t *, tok) {
+    /* This field is using GE(2) so for possible forward compatibility, we
+     * accept more fields but must be at least 2. */
+    tor_assert(tok->n_args >= 2);
+
+    /* Try to find an ntor key, it's the only recognized type right now */
+    if (!strcmp(tok->args[0], "ntor")) {
+      if (curve25519_public_from_base64(onion_key_out, tok->args[1]) < 0) {
+        log_warn(LD_REND, "Introduction point ntor onion-key is invalid");
+        goto err;
+      }
+      /* Got the onion key! Set the appropriate retval */
+      retval = 0;
+    }
+  } SMARTLIST_FOREACH_END(tok);
+
+  /* Log an error if we didn't find it :( */
+  if (retval < 0) {
+    log_warn(LD_REND, "Descriptor did not contain ntor onion keys");
+  }
+
+ err:
+  smartlist_free(onion_keys);
+  return retval;
+}
+
+/* Given the start of a section and the end of it, decode a single
+ * introduction point from that section. Return a newly allocated introduction
+ * point object containing the decoded data. Return NULL if the section can't
+ * be decoded. */
+STATIC hs_desc_intro_point_t *
+decode_introduction_point(const hs_descriptor_t *desc, const char *start)
+{
+  hs_desc_intro_point_t *ip = NULL;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+  const directory_token_t *tok;
+
+  tor_assert(desc);
+  tor_assert(start);
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, start, start + strlen(start),
+                      tokens, hs_desc_intro_point_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Introduction point is not parseable");
+    goto err;
+  }
+
+  /* Ok we seem to have a well formed section containing enough tokens to
+   * parse. Allocate our IP object and try to populate it. */
+  ip = hs_desc_intro_point_new();
+
+  /* "introduction-point" SP link-specifiers NL */
+  tok = find_by_keyword(tokens, R3_INTRODUCTION_POINT);
+  tor_assert(tok->n_args == 1);
+  /* Our constructor creates this list by default so free it. */
+  smartlist_free(ip->link_specifiers);
+  ip->link_specifiers = decode_link_specifiers(tok->args[0]);
+  if (!ip->link_specifiers) {
+    log_warn(LD_REND, "Introduction point has invalid link specifiers");
+    goto err;
+  }
+
+  /* "onion-key" SP ntor SP key NL */
+  if (set_intro_point_onion_key(&ip->onion_key, tokens) < 0) {
+    goto err;
+  }
+
+  /* "auth-key" NL certificate NL */
+  tok = find_by_keyword(tokens, R3_INTRO_AUTH_KEY);
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "ED25519 CERT")) {
+    log_warn(LD_REND, "Unexpected object type for introduction auth key");
+    goto err;
+  }
+  /* Parse cert and do some validation. */
+  if (cert_parse_and_validate(&ip->auth_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_AUTH_HS_IP_KEY,
+                              "introduction point auth-key") < 0) {
+    goto err;
+  }
+  /* Validate authentication certificate with descriptor signing key. */
+  if (tor_cert_checksig(ip->auth_key_cert,
+                        &desc->plaintext_data.signing_pubkey, 0) < 0) {
+    log_warn(LD_REND, "Invalid authentication key signature: %s",
+             tor_cert_describe_signature_status(ip->auth_key_cert));
+    goto err;
+  }
+
+  /* Exactly one "enc-key" SP "ntor" SP key NL */
+  tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY);
+  if (!strcmp(tok->args[0], "ntor")) {
+    /* This field is using GE(2) so for possible forward compatibility, we
+     * accept more fields but must be at least 2. */
+    tor_assert(tok->n_args >= 2);
+
+    if (curve25519_public_from_base64(&ip->enc_key, tok->args[1]) < 0) {
+      log_warn(LD_REND, "Introduction point ntor enc-key is invalid");
+      goto err;
+    }
+  } else {
+    /* Unknown key type so we can't use that introduction point. */
+    log_warn(LD_REND, "Introduction point encryption key is unrecognized.");
+    goto err;
+  }
+
+  /* Exactly once "enc-key-cert" NL certificate NL */
+  tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY_CERT);
+  tor_assert(tok->object_body);
+  /* Do the cross certification. */
+  if (strcmp(tok->object_type, "ED25519 CERT")) {
+      log_warn(LD_REND, "Introduction point ntor encryption key "
+                        "cross-certification has an unknown format.");
+      goto err;
+  }
+  if (cert_parse_and_validate(&ip->enc_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_CROSS_HS_IP_KEYS,
+                              "introduction point enc-key-cert") < 0) {
+    goto err;
+  }
+  if (tor_cert_checksig(ip->enc_key_cert,
+                        &desc->plaintext_data.signing_pubkey, 0) < 0) {
+    log_warn(LD_REND, "Invalid encryption key signature: %s",
+             tor_cert_describe_signature_status(ip->enc_key_cert));
+    goto err;
+  }
+  /* It is successfully cross certified. Flag the object. */
+  ip->cross_certified = 1;
+
+  /* Do we have a "legacy-key" SP key NL ?*/
+  tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY);
+  if (tok) {
+    if (decode_intro_legacy_key(tok, tokens, ip, desc) < 0) {
+      goto err;
+    }
+  }
+
+  /* Introduction point has been parsed successfully. */
+  goto done;
+
+ err:
+  hs_desc_intro_point_free(ip);
+  ip = NULL;
+
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    memarea_drop_all(area);
+  }
+
+  return ip;
+}
+
+/* Given a descriptor string at <b>data</b>, decode all possible introduction
+ * points that we can find. Add the introduction point object to desc_enc as we
+ * find them. This function can't fail and it is possible that zero
+ * introduction points can be decoded. */
+static void
+decode_intro_points(const hs_descriptor_t *desc,
+                    hs_desc_encrypted_data_t *desc_enc,
+                    const char *data)
+{
+  smartlist_t *chunked_desc = smartlist_new();
+  smartlist_t *intro_points = smartlist_new();
+
+  tor_assert(desc);
+  tor_assert(desc_enc);
+  tor_assert(data);
+  tor_assert(desc_enc->intro_points);
+
+  /* Take the desc string, and extract the intro point substrings out of it */
+  {
+    /* Split the descriptor string using the intro point header as delimiter */
+    smartlist_split_string(chunked_desc, data, str_intro_point_start, 0, 0);
+
+    /* Check if there are actually any intro points included. The first chunk
+     * should be other descriptor fields (e.g. create2-formats), so it's not an
+     * intro point. */
+    if (smartlist_len(chunked_desc) < 2) {
+      goto done;
+    }
+  }
+
+  /* Take the intro point substrings, and prepare them for parsing */
+  {
+    int i = 0;
+    /* Prepend the introduction-point header to all the chunks, since
+       smartlist_split_string() devoured it. */
+    SMARTLIST_FOREACH_BEGIN(chunked_desc, char *, chunk) {
+      /* Ignore first chunk. It's other descriptor fields. */
+      if (i++ == 0) {
+        continue;
+      }
+
+      smartlist_add_asprintf(intro_points, "%s %s", str_intro_point, chunk);
+    } SMARTLIST_FOREACH_END(chunk);
+  }
+
+  /* Parse the intro points! */
+  SMARTLIST_FOREACH_BEGIN(intro_points, const char *, intro_point) {
+    hs_desc_intro_point_t *ip = decode_introduction_point(desc, intro_point);
+    if (!ip) {
+      /* Malformed introduction point section. We'll ignore this introduction
+       * point and continue parsing. New or unknown fields are possible for
+       * forward compatibility. */
+      continue;
+    }
+    smartlist_add(desc_enc->intro_points, ip);
+  } SMARTLIST_FOREACH_END(intro_point);
+
+ done:
+  SMARTLIST_FOREACH(chunked_desc, char *, a, tor_free(a));
+  smartlist_free(chunked_desc);
+  SMARTLIST_FOREACH(intro_points, char *, a, tor_free(a));
+  smartlist_free(intro_points);
+}
+/* Return 1 iff the given base64 encoded signature in b64_sig from the encoded
+ * descriptor in encoded_desc validates the descriptor content. */
+STATIC int
+desc_sig_is_valid(const char *b64_sig,
+                  const ed25519_public_key_t *signing_pubkey,
+                  const char *encoded_desc, size_t encoded_len)
+{
+  int ret = 0;
+  ed25519_signature_t sig;
+  const char *sig_start;
+
+  tor_assert(b64_sig);
+  tor_assert(signing_pubkey);
+  tor_assert(encoded_desc);
+  /* Verifying nothing won't end well :). */
+  tor_assert(encoded_len > 0);
+
+  /* Signature length check. */
+  if (strlen(b64_sig) != ED25519_SIG_BASE64_LEN) {
+    log_warn(LD_REND, "Service descriptor has an invalid signature length."
+                      "Exptected %d but got %lu",
+             ED25519_SIG_BASE64_LEN, (unsigned long) strlen(b64_sig));
+    goto err;
+  }
+
+  /* First, convert base64 blob to an ed25519 signature. */
+  if (ed25519_signature_from_base64(&sig, b64_sig) != 0) {
+    log_warn(LD_REND, "Service descriptor does not contain a valid "
+                      "signature");
+    goto err;
+  }
+
+  /* Find the start of signature. */
+  sig_start = tor_memstr(encoded_desc, encoded_len, "\n" str_signature " ");
+  /* Getting here means the token parsing worked for the signature so if we
+   * can't find the start of the signature, we have a code flow issue. */
+  if (!sig_start) {
+    log_warn(LD_GENERAL, "Malformed signature line. Rejecting.");
+    goto err;
+  }
+  /* Skip newline, it has to go in the signature check. */
+  sig_start++;
+
+  /* Validate signature with the full body of the descriptor. */
+  if (ed25519_checksig_prefixed(&sig,
+                                (const uint8_t *) encoded_desc,
+                                sig_start - encoded_desc,
+                                str_desc_sig_prefix,
+                                signing_pubkey) != 0) {
+    log_warn(LD_REND, "Invalid signature on service descriptor");
+    goto err;
+  }
+  /* Valid signature! All is good. */
+  ret = 1;
+
+ err:
+  return ret;
+}
+
+/* Decode descriptor plaintext data for version 3. Given a list of tokens, an
+ * allocated plaintext object that will be populated and the encoded
+ * descriptor with its length. The last one is needed for signature
+ * verification. Unknown tokens are simply ignored so this won't error on
+ * unknowns but requires that all v3 token be present and valid.
+ *
+ * Return 0 on success else a negative value. */
+static int
+desc_decode_plaintext_v3(smartlist_t *tokens,
+                         hs_desc_plaintext_data_t *desc,
+                         const char *encoded_desc, size_t encoded_len)
+{
+  int ok;
+  directory_token_t *tok;
+
+  tor_assert(tokens);
+  tor_assert(desc);
+  /* Version higher could still use this function to decode most of the
+   * descriptor and then they decode the extra part. */
+  tor_assert(desc->version >= 3);
+
+  /* Descriptor lifetime parsing. */
+  tok = find_by_keyword(tokens, R3_DESC_LIFETIME);
+  tor_assert(tok->n_args == 1);
+  desc->lifetime_sec = (uint32_t) tor_parse_ulong(tok->args[0], 10, 0,
+                                                  UINT32_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor lifetime value is invalid");
+    goto err;
+  }
+  /* Put it from minute to second. */
+  desc->lifetime_sec *= 60;
+  if (desc->lifetime_sec > HS_DESC_MAX_LIFETIME) {
+    log_warn(LD_REND, "Service descriptor lifetime is too big. "
+                      "Got %" PRIu32 " but max is %d",
+             desc->lifetime_sec, HS_DESC_MAX_LIFETIME);
+    goto err;
+  }
+
+  /* Descriptor signing certificate. */
+  tok = find_by_keyword(tokens, R3_DESC_SIGNING_CERT);
+  tor_assert(tok->object_body);
+  /* Expecting a prop220 cert with the signing key extension, which contains
+   * the blinded public key. */
+  if (strcmp(tok->object_type, "ED25519 CERT") != 0) {
+    log_warn(LD_REND, "Service descriptor signing cert wrong type (%s)",
+             escaped(tok->object_type));
+    goto err;
+  }
+  if (cert_parse_and_validate(&desc->signing_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_SIGNING_HS_DESC,
+                              "service descriptor signing key") < 0) {
+    goto err;
+  }
+
+  /* Copy the public keys into signing_pubkey and blinded_pubkey */
+  memcpy(&desc->signing_pubkey, &desc->signing_key_cert->signed_key,
+         sizeof(ed25519_public_key_t));
+  memcpy(&desc->blinded_pubkey, &desc->signing_key_cert->signing_key,
+         sizeof(ed25519_public_key_t));
+
+  /* Extract revision counter value. */
+  tok = find_by_keyword(tokens, R3_REVISION_COUNTER);
+  tor_assert(tok->n_args == 1);
+  desc->revision_counter = tor_parse_uint64(tok->args[0], 10, 0,
+                                            UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor revision-counter is invalid");
+    goto err;
+  }
+
+  /* Extract the encrypted data section. */
+  tok = find_by_keyword(tokens, R3_SUPERENCRYPTED);
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "MESSAGE") != 0) {
+    log_warn(LD_REND, "Service descriptor encrypted data section is invalid");
+    goto err;
+  }
+  /* Make sure the length of the encrypted blob is valid. */
+  if (!encrypted_data_length_is_valid(tok->object_size)) {
+    goto err;
+  }
+
+  /* Copy the encrypted blob to the descriptor object so we can handle it
+   * latter if needed. */
+  desc->superencrypted_blob = tor_memdup(tok->object_body, tok->object_size);
+  desc->superencrypted_blob_size = tok->object_size;
+
+  /* Extract signature and verify it. */
+  tok = find_by_keyword(tokens, R3_SIGNATURE);
+  tor_assert(tok->n_args == 1);
+  /* First arg here is the actual encoded signature. */
+  if (!desc_sig_is_valid(tok->args[0], &desc->signing_pubkey,
+                         encoded_desc, encoded_len)) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Decode the version 3 encrypted section of the given descriptor desc. The
+ * desc_encrypted_out will be populated with the decoded data. Return 0 on
+ * success else -1. */
+static int
+desc_decode_encrypted_v3(const hs_descriptor_t *desc,
+                         hs_desc_encrypted_data_t *desc_encrypted_out)
+{
+  int result = -1;
+  char *message = NULL;
+  size_t message_len;
+  memarea_t *area = NULL;
+  directory_token_t *tok;
+  smartlist_t *tokens = NULL;
+
+  tor_assert(desc);
+  tor_assert(desc_encrypted_out);
+
+  /* Decrypt the superencrypted data that is located in the plaintext section
+   * in the descriptor as a blob of bytes. */
+  message_len = desc_decrypt_all(desc, &message);
+  if (!message_len) {
+    log_warn(LD_REND, "Service descriptor decryption failed.");
+    goto err;
+  }
+  tor_assert(message);
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, message, message + message_len,
+                      tokens, hs_desc_encrypted_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Encrypted service descriptor is not parseable.");
+    goto err;
+  }
+
+  /* CREATE2 supported cell format. It's mandatory. */
+  tok = find_by_keyword(tokens, R3_CREATE2_FORMATS);
+  tor_assert(tok);
+  decode_create2_list(desc_encrypted_out, tok->args[0]);
+  /* Must support ntor according to the specification */
+  if (!desc_encrypted_out->create2_ntor) {
+    log_warn(LD_REND, "Service create2-formats does not include ntor.");
+    goto err;
+  }
+
+  /* Authentication type. It's optional but only once. */
+  tok = find_opt_by_keyword(tokens, R3_INTRO_AUTH_REQUIRED);
+  if (tok) {
+    if (!decode_auth_type(desc_encrypted_out, tok->args[0])) {
+      log_warn(LD_REND, "Service descriptor authentication type has "
+                        "invalid entry(ies).");
+      goto err;
+    }
+  }
+
+  /* Is this service a single onion service? */
+  tok = find_opt_by_keyword(tokens, R3_SINGLE_ONION_SERVICE);
+  if (tok) {
+    desc_encrypted_out->single_onion_service = 1;
+  }
+
+  /* Initialize the descriptor's introduction point list before we start
+   * decoding. Having 0 intro point is valid. Then decode them all. */
+  desc_encrypted_out->intro_points = smartlist_new();
+  decode_intro_points(desc, desc_encrypted_out, message);
+
+  /* Validation of maximum introduction points allowed. */
+  if (smartlist_len(desc_encrypted_out->intro_points) >
+      HS_CONFIG_V3_MAX_INTRO_POINTS) {
+    log_warn(LD_REND, "Service descriptor contains too many introduction "
+                      "points. Maximum allowed is %d but we have %d",
+             HS_CONFIG_V3_MAX_INTRO_POINTS,
+             smartlist_len(desc_encrypted_out->intro_points));
+    goto err;
+  }
+
+  /* NOTE: Unknown fields are allowed because this function could be used to
+   * decode other descriptor version. */
+
+  result = 0;
+  goto done;
+
+ err:
+  tor_assert(result < 0);
+  desc_encrypted_data_free_contents(desc_encrypted_out);
+
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    memarea_drop_all(area);
+  }
+  if (message) {
+    tor_free(message);
+  }
+  return result;
+}
+
+/* Table of encrypted decode function version specific. The function are
+ * indexed by the version number so v3 callback is at index 3 in the array. */
+static int
+  (*decode_encrypted_handlers[])(
+      const hs_descriptor_t *desc,
+      hs_desc_encrypted_data_t *desc_encrypted) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_decode_encrypted_v3,
+};
+
+/* Decode the encrypted data section of the given descriptor and store the
+ * data in the given encrypted data object. Return 0 on success else a
+ * negative value on error. */
+int
+hs_desc_decode_encrypted(const hs_descriptor_t *desc,
+                         hs_desc_encrypted_data_t *desc_encrypted)
+{
+  int ret;
+  uint32_t version;
+
+  tor_assert(desc);
+  /* Ease our life a bit. */
+  version = desc->plaintext_data.version;
+  tor_assert(desc_encrypted);
+  /* Calling this function without an encrypted blob to parse is a code flow
+   * error. The plaintext parsing should never succeed in the first place
+   * without an encrypted section. */
+  tor_assert(desc->plaintext_data.superencrypted_blob);
+  /* Let's make sure we have a supported version as well. By correctly parsing
+   * the plaintext, this should not fail. */
+  if (BUG(!hs_desc_is_supported_version(version))) {
+    ret = -1;
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(decode_encrypted_handlers) >= version);
+  tor_assert(decode_encrypted_handlers[version]);
+
+  /* Run the version specific plaintext decoder. */
+  ret = decode_encrypted_handlers[version](desc, desc_encrypted);
+  if (ret < 0) {
+    goto err;
+  }
+
+ err:
+  return ret;
+}
+
+/* Table of plaintext decode function version specific. The function are
+ * indexed by the version number so v3 callback is at index 3 in the array. */
+static int
+  (*decode_plaintext_handlers[])(
+      smartlist_t *tokens,
+      hs_desc_plaintext_data_t *desc,
+      const char *encoded_desc,
+      size_t encoded_len) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_decode_plaintext_v3,
+};
+
+/* Fully decode the given descriptor plaintext and store the data in the
+ * plaintext data object. Returns 0 on success else a negative value. */
+int
+hs_desc_decode_plaintext(const char *encoded,
+                         hs_desc_plaintext_data_t *plaintext)
+{
+  int ok = 0, ret = -1;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+  size_t encoded_len;
+  directory_token_t *tok;
+
+  tor_assert(encoded);
+  tor_assert(plaintext);
+
+  /* Check that descriptor is within size limits. */
+  encoded_len = strlen(encoded);
+  if (encoded_len >= hs_cache_get_max_descriptor_size()) {
+    log_warn(LD_REND, "Service descriptor is too big (%lu bytes)",
+             (unsigned long) encoded_len);
+    goto err;
+  }
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  /* Tokenize the descriptor so we can start to parse it. */
+  if (tokenize_string(area, encoded, encoded + encoded_len, tokens,
+                      hs_desc_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Service descriptor is not parseable");
+    goto err;
+  }
+
+  /* Get the version of the descriptor which is the first mandatory field of
+   * the descriptor. From there, we'll decode the right descriptor version. */
+  tok = find_by_keyword(tokens, R_HS_DESCRIPTOR);
+  tor_assert(tok->n_args == 1);
+  plaintext->version = (uint32_t) tor_parse_ulong(tok->args[0], 10, 0,
+                                                  UINT32_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor has unparseable version %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  if (!hs_desc_is_supported_version(plaintext->version)) {
+    log_warn(LD_REND, "Service descriptor has unsupported version %" PRIu32,
+             plaintext->version);
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(decode_plaintext_handlers) >= plaintext->version);
+  tor_assert(decode_plaintext_handlers[plaintext->version]);
+
+  /* Run the version specific plaintext decoder. */
+  ret = decode_plaintext_handlers[plaintext->version](tokens, plaintext,
+                                                      encoded, encoded_len);
+  if (ret < 0) {
+    goto err;
+  }
+  /* Success. Descriptor has been populated with the data. */
+  ret = 0;
+
+ err:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    memarea_drop_all(area);
+  }
+  return ret;
+}
+
+/* Fully decode an encoded descriptor and set a newly allocated descriptor
+ * object in desc_out. Subcredentials are used if not NULL else it's ignored.
+ *
+ * Return 0 on success. A negative value is returned on error and desc_out is
+ * set to NULL. */
+int
+hs_desc_decode_descriptor(const char *encoded,
+                          const uint8_t *subcredential,
+                          hs_descriptor_t **desc_out)
+{
+  int ret = -1;
+  hs_descriptor_t *desc;
+
+  tor_assert(encoded);
+
+  desc = tor_malloc_zero(sizeof(hs_descriptor_t));
+
+  /* Subcredentials are optional. */
+  if (BUG(!subcredential)) {
+    log_warn(LD_GENERAL, "Tried to decrypt without subcred. Impossible!");
+    goto err;
+  }
+
+  memcpy(desc->subcredential, subcredential, sizeof(desc->subcredential));
+
+  ret = hs_desc_decode_plaintext(encoded, &desc->plaintext_data);
+  if (ret < 0) {
+    goto err;
+  }
+
+  ret = hs_desc_decode_encrypted(desc, &desc->encrypted_data);
+  if (ret < 0) {
+    goto err;
+  }
+
+  if (desc_out) {
+    *desc_out = desc;
+  } else {
+    hs_descriptor_free(desc);
+  }
+  return ret;
+
+ err:
+  hs_descriptor_free(desc);
+  if (desc_out) {
+    *desc_out = NULL;
+  }
+
+  tor_assert(ret < 0);
+  return ret;
+}
+
+/* Table of encode function version specific. The functions are indexed by the
+ * version number so v3 callback is at index 3 in the array. */
+static int
+  (*encode_handlers[])(
+      const hs_descriptor_t *desc,
+      const ed25519_keypair_t *signing_kp,
+      char **encoded_out) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_encode_v3,
+};
+
+/* Encode the given descriptor desc including signing with the given key pair
+ * signing_kp. On success, encoded_out points to a newly allocated NUL
+ * terminated string that contains the encoded descriptor as a string.
+ *
+ * Return 0 on success and encoded_out is a valid pointer. On error, -1 is
+ * returned and encoded_out is set to NULL. */
+MOCK_IMPL(int,
+hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
+                           const ed25519_keypair_t *signing_kp,
+                           char **encoded_out))
+{
+  int ret = -1;
+  uint32_t version;
+
+  tor_assert(desc);
+  tor_assert(encoded_out);
+
+  /* Make sure we support the version of the descriptor format. */
+  version = desc->plaintext_data.version;
+  if (!hs_desc_is_supported_version(version)) {
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(encode_handlers) >= version);
+  tor_assert(encode_handlers[version]);
+
+  ret = encode_handlers[version](desc, signing_kp, encoded_out);
+  if (ret < 0) {
+    goto err;
+  }
+
+  /* Try to decode what we just encoded. Symmetry is nice! */
+  ret = hs_desc_decode_descriptor(*encoded_out, desc->subcredential, NULL);
+  if (BUG(ret < 0)) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  *encoded_out = NULL;
+  return ret;
+}
+
+/* Free the descriptor plaintext data object. */
+void
+hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc)
+{
+  desc_plaintext_data_free_contents(desc);
+  tor_free(desc);
+}
+
+/* Free the descriptor encrypted data object. */
+void
+hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc)
+{
+  desc_encrypted_data_free_contents(desc);
+  tor_free(desc);
+}
+
+/* Free the given descriptor object. */
+void
+hs_descriptor_free_(hs_descriptor_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  desc_plaintext_data_free_contents(&desc->plaintext_data);
+  desc_encrypted_data_free_contents(&desc->encrypted_data);
+  tor_free(desc);
+}
+
+/* Return the size in bytes of the given plaintext data object. A sizeof() is
+ * not enough because the object contains pointers and the encrypted blob.
+ * This is particularly useful for our OOM subsystem that tracks the HSDir
+ * cache size for instance. */
+size_t
+hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data)
+{
+  tor_assert(data);
+  return (sizeof(*data) + sizeof(*data->signing_key_cert) +
+          data->superencrypted_blob_size);
+}
+
+/* Return the size in bytes of the given encrypted data object. Used by OOM
+ * subsystem. */
+static size_t
+hs_desc_encrypted_obj_size(const hs_desc_encrypted_data_t *data)
+{
+  tor_assert(data);
+  size_t intro_size = 0;
+  if (data->intro_auth_types) {
+    intro_size +=
+      smartlist_len(data->intro_auth_types) * sizeof(intro_auth_types);
+  }
+  if (data->intro_points) {
+    /* XXX could follow pointers here and get more accurate size */
+    intro_size +=
+      smartlist_len(data->intro_points) * sizeof(hs_desc_intro_point_t);
+  }
+
+  return sizeof(*data) + intro_size;
+}
+
+/* Return the size in bytes of the given descriptor object. Used by OOM
+ * subsystem. */
+  size_t
+hs_desc_obj_size(const hs_descriptor_t *data)
+{
+  tor_assert(data);
+  return (hs_desc_plaintext_obj_size(&data->plaintext_data) +
+          hs_desc_encrypted_obj_size(&data->encrypted_data) +
+          sizeof(data->subcredential));
+}
+
+/* Return a newly allocated descriptor intro point. */
+hs_desc_intro_point_t *
+hs_desc_intro_point_new(void)
+{
+  hs_desc_intro_point_t *ip = tor_malloc_zero(sizeof(*ip));
+  ip->link_specifiers = smartlist_new();
+  return ip;
+}
+
+/* Free a descriptor intro point object. */
+void
+hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
+{
+  if (ip == NULL) {
+    return;
+  }
+  if (ip->link_specifiers) {
+    SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *,
+                      ls, hs_desc_link_specifier_free(ls));
+    smartlist_free(ip->link_specifiers);
+  }
+  tor_cert_free(ip->auth_key_cert);
+  tor_cert_free(ip->enc_key_cert);
+  crypto_pk_free(ip->legacy.key);
+  tor_free(ip->legacy.cert.encoded);
+  tor_free(ip);
+}
+
+/* Free the given descriptor link specifier. */
+void
+hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls)
+{
+  if (ls == NULL) {
+    return;
+  }
+  tor_free(ls);
+}
+
+/* Return a newly allocated descriptor link specifier using the given extend
+ * info and requested type. Return NULL on error. */
+hs_desc_link_specifier_t *
+hs_desc_link_specifier_new(const extend_info_t *info, uint8_t type)
+{
+  hs_desc_link_specifier_t *ls = NULL;
+
+  tor_assert(info);
+
+  ls = tor_malloc_zero(sizeof(*ls));
+  ls->type = type;
+  switch (ls->type) {
+  case LS_IPV4:
+    if (info->addr.family != AF_INET) {
+      goto err;
+    }
+    tor_addr_copy(&ls->u.ap.addr, &info->addr);
+    ls->u.ap.port = info->port;
+    break;
+  case LS_IPV6:
+    if (info->addr.family != AF_INET6) {
+      goto err;
+    }
+    tor_addr_copy(&ls->u.ap.addr, &info->addr);
+    ls->u.ap.port = info->port;
+    break;
+  case LS_LEGACY_ID:
+    /* Bug out if the identity digest is not set */
+    if (BUG(tor_mem_is_zero(info->identity_digest,
+                            sizeof(info->identity_digest)))) {
+      goto err;
+    }
+    memcpy(ls->u.legacy_id, info->identity_digest, sizeof(ls->u.legacy_id));
+    break;
+  case LS_ED25519_ID:
+    /* ed25519 keys are optional for intro points */
+    if (ed25519_public_key_is_zero(&info->ed_identity)) {
+      goto err;
+    }
+    memcpy(ls->u.ed25519_id, info->ed_identity.pubkey,
+           sizeof(ls->u.ed25519_id));
+    break;
+  default:
+    /* Unknown type is code flow error. */
+    tor_assert(0);
+  }
+
+  return ls;
+ err:
+  tor_free(ls);
+  return NULL;
+}
+
+/* From the given descriptor, remove and free every introduction point. */
+void
+hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
+{
+  smartlist_t *ips;
+
+  tor_assert(desc);
+
+  ips = desc->encrypted_data.intro_points;
+  if (ips) {
+    SMARTLIST_FOREACH(ips, hs_desc_intro_point_t *,
+                      ip, hs_desc_intro_point_free(ip));
+    smartlist_clear(ips);
+  }
+}
+
+/* From a descriptor link specifier object spec, returned a newly allocated
+ * link specifier object that is the encoded representation of spec. Return
+ * NULL on error. */
+link_specifier_t *
+hs_desc_lspec_to_trunnel(const hs_desc_link_specifier_t *spec)
+{
+  tor_assert(spec);
+
+  link_specifier_t *ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, spec->type);
+
+  switch (spec->type) {
+  case LS_IPV4:
+    link_specifier_set_un_ipv4_addr(ls,
+                                    tor_addr_to_ipv4h(&spec->u.ap.addr));
+    link_specifier_set_un_ipv4_port(ls, spec->u.ap.port);
+    /* Four bytes IPv4 and two bytes port. */
+    link_specifier_set_ls_len(ls, sizeof(spec->u.ap.addr.addr.in_addr) +
+                                  sizeof(spec->u.ap.port));
+    break;
+  case LS_IPV6:
+  {
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&spec->u.ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, spec->u.ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(spec->u.ap.port));
+    break;
+  }
+  case LS_LEGACY_ID:
+  {
+    size_t legacy_id_len = link_specifier_getlen_un_legacy_id(ls);
+    uint8_t *legacy_id_array = link_specifier_getarray_un_legacy_id(ls);
+    memcpy(legacy_id_array, spec->u.legacy_id, legacy_id_len);
+    link_specifier_set_ls_len(ls, legacy_id_len);
+    break;
+  }
+  case LS_ED25519_ID:
+  {
+    size_t ed25519_id_len = link_specifier_getlen_un_ed25519_id(ls);
+    uint8_t *ed25519_id_array = link_specifier_getarray_un_ed25519_id(ls);
+    memcpy(ed25519_id_array, spec->u.ed25519_id, ed25519_id_len);
+    link_specifier_set_ls_len(ls, ed25519_id_len);
+    break;
+  }
+  default:
+    tor_assert_nonfatal_unreached();
+    link_specifier_free(ls);
+    ls = NULL;
+  }
+
+  return ls;
+}
diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h
new file mode 100644
index 0000000000..7a4010cd3b
--- /dev/null
+++ b/src/feature/hs/hs_descriptor.h
@@ -0,0 +1,280 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_descriptor.h
+ * \brief Header file for hs_descriptor.c
+ **/
+
+#ifndef TOR_HS_DESCRIPTOR_H
+#define TOR_HS_DESCRIPTOR_H
+
+#include <stdint.h>
+
+#include "or/or.h"
+#include "trunnel/ed25519_cert.h" /* needed for trunnel */
+#include "or/torcert.h"
+
+/* Trunnel */
+struct link_specifier_t;
+
+/* The earliest descriptor format version we support. */
+#define HS_DESC_SUPPORTED_FORMAT_VERSION_MIN 3
+/* The latest descriptor format version we support. */
+#define HS_DESC_SUPPORTED_FORMAT_VERSION_MAX 3
+
+/* Default lifetime of a descriptor in seconds. The valus is set at 3 hours
+ * which is 180 minutes or 10800 seconds. */
+#define HS_DESC_DEFAULT_LIFETIME (3 * 60 * 60)
+/* Maximum lifetime of a descriptor in seconds. The value is set at 12 hours
+ * which is 720 minutes or 43200 seconds. */
+#define HS_DESC_MAX_LIFETIME (12 * 60 * 60)
+/* Lifetime of certificate in the descriptor. This defines the lifetime of the
+ * descriptor signing key and the cross certification cert of that key. It is
+ * set to 54 hours because a descriptor can be around for 48 hours and because
+ * consensuses are used after the hour, add an extra 6 hours to give some time
+ * for the service to stop using it. */
+#define HS_DESC_CERT_LIFETIME (54 * 60 * 60)
+/* Length of the salt needed for the encrypted section of a descriptor. */
+#define HS_DESC_ENCRYPTED_SALT_LEN 16
+/* Length of the secret input needed for the KDF construction which derives
+ * the encryption key for the encrypted data section of the descriptor. This
+ * adds up to 68 bytes being the blinded key, hashed subcredential and
+ * revision counter. */
+#define HS_DESC_ENCRYPTED_SECRET_INPUT_LEN \
+  ED25519_PUBKEY_LEN + DIGEST256_LEN + sizeof(uint64_t)
+/* Length of the KDF output value which is the length of the secret key,
+ * the secret IV and MAC key length which is the length of H() output. */
+#define HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN \
+  CIPHER256_KEY_LEN + CIPHER_IV_LEN + DIGEST256_LEN
+/* Pad plaintext of superencrypted data section before encryption so that its
+ * length is a multiple of this value. */
+#define HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE 10000
+/* Maximum length in bytes of a full hidden service descriptor. */
+#define HS_DESC_MAX_LEN 50000 /* 50kb max size */
+
+/* Key length for the descriptor symmetric encryption. As specified in the
+ * protocol, we use AES-256 for the encrypted section of the descriptor. The
+ * following is the length in bytes and the bit size. */
+#define HS_DESC_ENCRYPTED_KEY_LEN CIPHER256_KEY_LEN
+#define HS_DESC_ENCRYPTED_BIT_SIZE (HS_DESC_ENCRYPTED_KEY_LEN * 8)
+
+/* Type of authentication in the descriptor. */
+typedef enum {
+  HS_DESC_AUTH_ED25519 = 1
+} hs_desc_auth_type_t;
+
+/* Link specifier object that contains information on how to extend to the
+ * relay that is the address, port and handshake type. */
+typedef struct hs_desc_link_specifier_t {
+  /* Indicate the type of link specifier. See trunnel ed25519_cert
+   * specification. */
+  uint8_t type;
+
+  /* It must be one of these types, can't be more than one. */
+  union {
+    /* IP address and port of the relay use to extend. */
+    tor_addr_port_t ap;
+    /* Legacy identity. A 20-byte SHA1 identity fingerprint. */
+    uint8_t legacy_id[DIGEST_LEN];
+    /* ed25519 identity. A 32-byte key. */
+    uint8_t ed25519_id[ED25519_PUBKEY_LEN];
+  } u;
+} hs_desc_link_specifier_t;
+
+/* Introduction point information located in a descriptor. */
+typedef struct hs_desc_intro_point_t {
+  /* Link specifier(s) which details how to extend to the relay. This list
+   * contains hs_desc_link_specifier_t object. It MUST have at least one. */
+  smartlist_t *link_specifiers;
+
+  /* Onion key of the introduction point used to extend to it for the ntor
+   * handshake. */
+  curve25519_public_key_t onion_key;
+
+  /* Authentication key used to establish the introduction point circuit and
+   * cross-certifies the blinded public key for the replica thus signed by
+   * the blinded key and in turn signs it. */
+  tor_cert_t *auth_key_cert;
+
+  /* Encryption key for the "ntor" type. */
+  curve25519_public_key_t enc_key;
+
+  /* Certificate cross certifying the descriptor signing key by the encryption
+   * curve25519 key. This certificate contains the signing key and is of type
+   * CERT_TYPE_CROSS_HS_IP_KEYS [0B]. */
+  tor_cert_t *enc_key_cert;
+
+  /* (Optional): If this introduction point is a legacy one that is version <=
+   * 0.2.9.x (HSIntro=3), we use this extra key for the intro point to be able
+   * to relay the cells to the service correctly. */
+  struct {
+    /* RSA public key. */
+    crypto_pk_t *key;
+
+    /* Cross certified cert with the descriptor signing key (RSA->Ed). Because
+     * of the cross certification API, we need to keep the certificate binary
+     * blob and its length in order to properly encode it after. */
+    struct {
+      uint8_t *encoded;
+      size_t len;
+    } cert;
+  } legacy;
+
+  /* True iff the introduction point has passed the cross certification. Upon
+   * decoding an intro point, this must be true. */
+  unsigned int cross_certified : 1;
+} hs_desc_intro_point_t;
+
+/* The encrypted data section of a descriptor. Obviously the data in this is
+ * in plaintext but encrypted once encoded. */
+typedef struct hs_desc_encrypted_data_t {
+  /* Bitfield of CREATE2 cell supported formats. The only currently supported
+   * format is ntor. */
+  unsigned int create2_ntor : 1;
+
+  /* A list of authentication types that a client must at least support one
+   * in order to contact the service. Contains NULL terminated strings. */
+  smartlist_t *intro_auth_types;
+
+  /* Is this descriptor a single onion service? */
+  unsigned int single_onion_service : 1;
+
+  /* A list of intro points. Contains hs_desc_intro_point_t objects. */
+  smartlist_t *intro_points;
+} hs_desc_encrypted_data_t;
+
+/* Plaintext data that is unencrypted information of the descriptor. */
+typedef struct hs_desc_plaintext_data_t {
+  /* Version of the descriptor format. Spec specifies this field as a
+   * positive integer. */
+  uint32_t version;
+
+  /* The lifetime of the descriptor in seconds. */
+  uint32_t lifetime_sec;
+
+  /* Certificate with the short-term ed22519 descriptor signing key for the
+   * replica which is signed by the blinded public key for that replica. */
+  tor_cert_t *signing_key_cert;
+
+  /* Signing public key which is used to sign the descriptor. Same public key
+   * as in the signing key certificate. */
+  ed25519_public_key_t signing_pubkey;
+
+  /* Blinded public key used for this descriptor derived from the master
+   * identity key and generated for a specific replica number. */
+  ed25519_public_key_t blinded_pubkey;
+
+  /* Revision counter is incremented at each upload, regardless of whether
+   * the descriptor has changed. This avoids leaking whether the descriptor
+   * has changed. Spec specifies this as a 8 bytes positive integer. */
+  uint64_t revision_counter;
+
+  /* Decoding only: The b64-decoded superencrypted blob from the descriptor */
+  uint8_t *superencrypted_blob;
+
+  /* Decoding only: Size of the superencrypted_blob */
+  size_t superencrypted_blob_size;
+} hs_desc_plaintext_data_t;
+
+/* Service descriptor in its decoded form. */
+typedef struct hs_descriptor_t {
+  /* Contains the plaintext part of the descriptor. */
+  hs_desc_plaintext_data_t plaintext_data;
+
+  /* The following contains what's in the encrypted part of the descriptor.
+   * It's only encrypted in the encoded version of the descriptor thus the
+   * data contained in that object is in plaintext. */
+  hs_desc_encrypted_data_t encrypted_data;
+
+  /* Subcredentials of a service, used by the client and service to decrypt
+   * the encrypted data. */
+  uint8_t subcredential[DIGEST256_LEN];
+} hs_descriptor_t;
+
+/* Return true iff the given descriptor format version is supported. */
+static inline int
+hs_desc_is_supported_version(uint32_t version)
+{
+  if (version < HS_DESC_SUPPORTED_FORMAT_VERSION_MIN ||
+      version > HS_DESC_SUPPORTED_FORMAT_VERSION_MAX) {
+    return 0;
+  }
+  return 1;
+}
+
+/* Public API. */
+
+void hs_descriptor_free_(hs_descriptor_t *desc);
+#define hs_descriptor_free(desc) \
+  FREE_AND_NULL(hs_descriptor_t, hs_descriptor_free_, (desc))
+void hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc);
+#define hs_desc_plaintext_data_free(desc) \
+  FREE_AND_NULL(hs_desc_plaintext_data_t, hs_desc_plaintext_data_free_, (desc))
+void hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc);
+#define hs_desc_encrypted_data_free(desc) \
+  FREE_AND_NULL(hs_desc_encrypted_data_t, hs_desc_encrypted_data_free_, (desc))
+
+void hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls);
+#define hs_desc_link_specifier_free(ls) \
+  FREE_AND_NULL(hs_desc_link_specifier_t, hs_desc_link_specifier_free_, (ls))
+
+hs_desc_link_specifier_t *hs_desc_link_specifier_new(
+                                  const extend_info_t *info, uint8_t type);
+void hs_descriptor_clear_intro_points(hs_descriptor_t *desc);
+
+MOCK_DECL(int,
+          hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
+                                     const ed25519_keypair_t *signing_kp,
+                                     char **encoded_out));
+
+int hs_desc_decode_descriptor(const char *encoded,
+                              const uint8_t *subcredential,
+                              hs_descriptor_t **desc_out);
+int hs_desc_decode_plaintext(const char *encoded,
+                             hs_desc_plaintext_data_t *plaintext);
+int hs_desc_decode_encrypted(const hs_descriptor_t *desc,
+                             hs_desc_encrypted_data_t *desc_out);
+
+size_t hs_desc_obj_size(const hs_descriptor_t *data);
+size_t hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data);
+
+hs_desc_intro_point_t *hs_desc_intro_point_new(void);
+void hs_desc_intro_point_free_(hs_desc_intro_point_t *ip);
+#define hs_desc_intro_point_free(ip) \
+  FREE_AND_NULL(hs_desc_intro_point_t, hs_desc_intro_point_free_, (ip))
+
+link_specifier_t *hs_desc_lspec_to_trunnel(
+                                   const hs_desc_link_specifier_t *spec);
+
+#ifdef HS_DESCRIPTOR_PRIVATE
+
+/* Encoding. */
+STATIC char *encode_link_specifiers(const smartlist_t *specs);
+STATIC size_t build_plaintext_padding(const char *plaintext,
+                                      size_t plaintext_len,
+                                      uint8_t **padded_out);
+/* Decoding. */
+STATIC smartlist_t *decode_link_specifiers(const char *encoded);
+STATIC hs_desc_intro_point_t *decode_introduction_point(
+                                const hs_descriptor_t *desc,
+                                const char *text);
+STATIC int encrypted_data_length_is_valid(size_t len);
+STATIC int cert_is_valid(tor_cert_t *cert, uint8_t type,
+                         const char *log_obj_type);
+STATIC int desc_sig_is_valid(const char *b64_sig,
+                             const ed25519_public_key_t *signing_pubkey,
+                             const char *encoded_desc, size_t encoded_len);
+STATIC size_t decode_superencrypted(const char *message, size_t message_len,
+                                   uint8_t **encrypted_out);
+STATIC void desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc);
+
+MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc,
+                                             const uint8_t *encrypted_blob,
+                                             size_t encrypted_blob_size,
+                                             int is_superencrypted_layer,
+                                             char **decrypted_out));
+
+#endif /* defined(HS_DESCRIPTOR_PRIVATE) */
+
+#endif /* !defined(TOR_HS_DESCRIPTOR_H) */
diff --git a/src/feature/hs/hs_ident.c b/src/feature/hs/hs_ident.c
new file mode 100644
index 0000000000..20539ca878
--- /dev/null
+++ b/src/feature/hs/hs_ident.c
@@ -0,0 +1,127 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ident.c
+ * \brief Contains circuit and connection identifier code for the whole HS
+ *        subsytem.
+ **/
+
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/hs_ident.h"
+
+/* Return a newly allocated circuit identifier. The given public key is copied
+ * identity_pk into the identifier. */
+hs_ident_circuit_t *
+hs_ident_circuit_new(const ed25519_public_key_t *identity_pk,
+                     hs_ident_circuit_type_t circuit_type)
+{
+  tor_assert(circuit_type == HS_IDENT_CIRCUIT_INTRO ||
+             circuit_type == HS_IDENT_CIRCUIT_RENDEZVOUS);
+  hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
+  ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
+  ident->circuit_type = circuit_type;
+  return ident;
+}
+
+/* Free the given circuit identifier. */
+void
+hs_ident_circuit_free_(hs_ident_circuit_t *ident)
+{
+  if (ident == NULL) {
+    return;
+  }
+  memwipe(ident, 0, sizeof(hs_ident_circuit_t));
+  tor_free(ident);
+}
+
+/* For a given circuit identifier src, return a newly allocated copy of it.
+ * This can't fail. */
+hs_ident_circuit_t *
+hs_ident_circuit_dup(const hs_ident_circuit_t *src)
+{
+  hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
+  memcpy(ident, src, sizeof(*ident));
+  return ident;
+}
+
+/* For a given directory connection identifier src, return a newly allocated
+ * copy of it. This can't fail. */
+hs_ident_dir_conn_t *
+hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src)
+{
+  hs_ident_dir_conn_t *ident = tor_malloc_zero(sizeof(*ident));
+  memcpy(ident, src, sizeof(*ident));
+  return ident;
+}
+
+/* Free the given directory connection identifier. */
+void
+hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident)
+{
+  if (ident == NULL) {
+    return;
+  }
+  memwipe(ident, 0, sizeof(hs_ident_dir_conn_t));
+  tor_free(ident);
+}
+
+/* Initialized the allocated ident object with identity_pk and blinded_pk.
+ * None of them can be NULL since a valid directory connection identifier must
+ * have all fields set. */
+void
+hs_ident_dir_conn_init(const ed25519_public_key_t *identity_pk,
+                       const ed25519_public_key_t *blinded_pk,
+                       hs_ident_dir_conn_t *ident)
+{
+  tor_assert(identity_pk);
+  tor_assert(blinded_pk);
+  tor_assert(ident);
+
+  ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
+  ed25519_pubkey_copy(&ident->blinded_pk, blinded_pk);
+}
+
+/* Return a newly allocated edge connection identifier. The given public key
+ * identity_pk is copied into the identifier. */
+hs_ident_edge_conn_t *
+hs_ident_edge_conn_new(const ed25519_public_key_t *identity_pk)
+{
+  hs_ident_edge_conn_t *ident = tor_malloc_zero(sizeof(*ident));
+  ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
+  return ident;
+}
+
+/* Free the given edge connection identifier. */
+void
+hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident)
+{
+  if (ident == NULL) {
+    return;
+  }
+  memwipe(ident, 0, sizeof(hs_ident_edge_conn_t));
+  tor_free(ident);
+}
+
+/* Return true if the given ident is valid for an introduction circuit. */
+int
+hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident)
+{
+  if (ident == NULL) {
+    goto invalid;
+  }
+
+  if (ed25519_public_key_is_zero(&ident->identity_pk)) {
+    goto invalid;
+  }
+
+  if (ed25519_public_key_is_zero(&ident->intro_auth_pk)) {
+    goto invalid;
+  }
+
+  /* Valid. */
+  return 1;
+ invalid:
+  return 0;
+}
+
diff --git a/src/feature/hs/hs_ident.h b/src/feature/hs/hs_ident.h
new file mode 100644
index 0000000000..8c53b9dbeb
--- /dev/null
+++ b/src/feature/hs/hs_ident.h
@@ -0,0 +1,146 @@
+/* Copyright (c) 2017-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ident.h
+ * \brief Header file containing circuit and connection identifier data for
+ *        the whole HS subsytem.
+ *
+ * \details
+ * This interface is used to uniquely identify a hidden service on a circuit
+ * or connection using the service identity public key. Once the circuit or
+ * connection subsystem calls in the hidden service one, we use those
+ * identifiers to lookup the corresponding objects like service, intro point
+ * and descriptor.
+ *
+ * Furthermore, the circuit identifier holds cryptographic material needed for
+ * the e2e encryption on the rendezvous circuit which is set once the
+ * rendezvous circuit has opened and ready to be used.
+ **/
+
+#ifndef TOR_HS_IDENT_H
+#define TOR_HS_IDENT_H
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "or/hs_common.h"
+
+/* Length of the rendezvous cookie that is used to connect circuits at the
+ * rendezvous point. */
+#define HS_REND_COOKIE_LEN DIGEST_LEN
+
+/* Type of circuit an hs_ident_t object is associated with. */
+typedef enum {
+  HS_IDENT_CIRCUIT_INTRO      = 1,
+  HS_IDENT_CIRCUIT_RENDEZVOUS = 2,
+} hs_ident_circuit_type_t;
+
+/* Client and service side circuit identifier that is used for hidden service
+ * circuit establishment. Not all fields contain data, it depends on the
+ * circuit purpose. This is attached to an origin_circuit_t. All fields are
+ * used by both client and service. */
+typedef struct hs_ident_circuit_t {
+  /* (All circuit) The public key used to uniquely identify the service. It is
+   * the one found in the onion address. */
+  ed25519_public_key_t identity_pk;
+
+  /* (All circuit) The type of circuit this identifier is attached to.
+   * Accessors of the fields in this object assert non fatal on this circuit
+   * type. In other words, if a rendezvous field is being accessed, the
+   * circuit type MUST BE of type HS_IDENT_CIRCUIT_RENDEZVOUS. This value is
+   * set when an object is initialized in its constructor. */
+  hs_ident_circuit_type_t circuit_type;
+
+  /* (All circuit) Introduction point authentication key. It's also needed on
+   * the rendezvous circuit for the ntor handshake. It's used as the unique key
+   * of the introduction point so it should not be shared between multiple
+   * intro points. */
+  ed25519_public_key_t intro_auth_pk;
+
+  /* (Only client rendezvous circuit) Introduction point encryption public
+   * key. We keep it in the rendezvous identifier for the ntor handshake. */
+  curve25519_public_key_t intro_enc_pk;
+
+  /* (Only rendezvous circuit) Rendezvous cookie sent from the client to the
+   * service with an INTRODUCE1 cell and used by the service in an
+   * RENDEZVOUS1 cell. */
+  uint8_t rendezvous_cookie[HS_REND_COOKIE_LEN];
+
+  /* (Only service rendezvous circuit) The HANDSHAKE_INFO needed in the
+   * RENDEZVOUS1 cell of the service. The construction is as follows:
+   *    SERVER_PK   [32 bytes]
+   *    AUTH_MAC    [32 bytes]
+   */
+  uint8_t rendezvous_handshake_info[CURVE25519_PUBKEY_LEN + DIGEST256_LEN];
+
+  /* (Only client rendezvous circuit) Client ephemeral keypair needed for the
+   * e2e encryption with the service. */
+  curve25519_keypair_t rendezvous_client_kp;
+
+  /* (Only rendezvous circuit) The NTOR_KEY_SEED needed for key derivation for
+   * the e2e encryption with the client on the circuit. */
+  uint8_t rendezvous_ntor_key_seed[DIGEST256_LEN];
+
+  /* (Only rendezvous circuit) Number of streams associated with this
+   * rendezvous circuit. We track this because there is a check on a maximum
+   * value. */
+  uint64_t num_rdv_streams;
+} hs_ident_circuit_t;
+
+/* Client and service side directory connection identifier used for a
+ * directory connection to identify which service is being queried. This is
+ * attached to a dir_connection_t. */
+typedef struct hs_ident_dir_conn_t {
+  /* The public key used to uniquely identify the service. It is the one found
+   * in the onion address. */
+  ed25519_public_key_t identity_pk;
+
+  /* The blinded public key used to uniquely identify the descriptor that this
+   * directory connection identifier is for. Only used by the service-side code
+   * to fine control descriptor uploads. */
+  ed25519_public_key_t blinded_pk;
+
+  /* XXX: Client authorization. */
+} hs_ident_dir_conn_t;
+
+/* Client and service side edge connection identifier used for an edge
+ * connection to identify which service is being queried. This is attached to
+ * a edge_connection_t. */
+typedef struct hs_ident_edge_conn_t {
+  /* The public key used to uniquely identify the service. It is the one found
+   * in the onion address. */
+  ed25519_public_key_t identity_pk;
+
+  /* XXX: Client authorization. */
+} hs_ident_edge_conn_t;
+
+/* Circuit identifier API. */
+hs_ident_circuit_t *hs_ident_circuit_new(
+                             const ed25519_public_key_t *identity_pk,
+                             hs_ident_circuit_type_t circuit_type);
+void hs_ident_circuit_free_(hs_ident_circuit_t *ident);
+#define hs_ident_circuit_free(id) \
+  FREE_AND_NULL(hs_ident_circuit_t, hs_ident_circuit_free_, (id))
+hs_ident_circuit_t *hs_ident_circuit_dup(const hs_ident_circuit_t *src);
+
+/* Directory connection identifier API. */
+hs_ident_dir_conn_t *hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src);
+void hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident);
+#define hs_ident_dir_conn_free(id) \
+  FREE_AND_NULL(hs_ident_dir_conn_t, hs_ident_dir_conn_free_, (id))
+void hs_ident_dir_conn_init(const ed25519_public_key_t *identity_pk,
+                            const ed25519_public_key_t *blinded_pk,
+                            hs_ident_dir_conn_t *ident);
+
+/* Edge connection identifier API. */
+hs_ident_edge_conn_t *hs_ident_edge_conn_new(
+                                    const ed25519_public_key_t *identity_pk);
+void hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident);
+#define hs_ident_edge_conn_free(id) \
+  FREE_AND_NULL(hs_ident_edge_conn_t, hs_ident_edge_conn_free_, (id))
+
+/* Validators */
+int hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident);
+
+#endif /* !defined(TOR_HS_IDENT_H) */
+
diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c
new file mode 100644
index 0000000000..2594058679
--- /dev/null
+++ b/src/feature/hs/hs_intropoint.c
@@ -0,0 +1,614 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_intropoint.c
+ * \brief Implement next generation introductions point functionality
+ **/
+
+#define HS_INTROPOINT_PRIVATE
+
+#include "or/or.h"
+#include "or/config.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/relay.h"
+#include "or/rendmid.h"
+#include "or/rephist.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+/* Trunnel */
+#include "trunnel/ed25519_cert.h"
+#include "trunnel/hs/cell_common.h"
+#include "trunnel/hs/cell_establish_intro.h"
+#include "trunnel/hs/cell_introduce1.h"
+
+#include "or/hs_circuitmap.h"
+#include "or/hs_descriptor.h"
+#include "or/hs_intropoint.h"
+#include "or/hs_common.h"
+
+#include "or/or_circuit_st.h"
+
+/** Extract the authentication key from an ESTABLISH_INTRO or INTRODUCE1 using
+ * the given <b>cell_type</b> from <b>cell</b> and place it in
+ * <b>auth_key_out</b>. */
+STATIC void
+get_auth_key_from_cell(ed25519_public_key_t *auth_key_out,
+                       unsigned int cell_type, const void *cell)
+{
+  size_t auth_key_len;
+  const uint8_t *key_array;
+
+  tor_assert(auth_key_out);
+  tor_assert(cell);
+
+  switch (cell_type) {
+  case RELAY_COMMAND_ESTABLISH_INTRO:
+  {
+    const trn_cell_establish_intro_t *c_cell = cell;
+    key_array = trn_cell_establish_intro_getconstarray_auth_key(c_cell);
+    auth_key_len = trn_cell_establish_intro_getlen_auth_key(c_cell);
+    break;
+  }
+  case RELAY_COMMAND_INTRODUCE1:
+  {
+    const trn_cell_introduce1_t *c_cell = cell;
+    key_array = trn_cell_introduce1_getconstarray_auth_key(cell);
+    auth_key_len = trn_cell_introduce1_getlen_auth_key(c_cell);
+    break;
+  }
+  default:
+    /* Getting here is really bad as it means we got a unknown cell type from
+     * this file where every call has an hardcoded value. */
+    tor_assert(0); /* LCOV_EXCL_LINE */
+  }
+  tor_assert(key_array);
+  tor_assert(auth_key_len == sizeof(auth_key_out->pubkey));
+  memcpy(auth_key_out->pubkey, key_array, auth_key_len);
+}
+
+/** We received an ESTABLISH_INTRO <b>cell</b>. Verify its signature and MAC,
+ *  given <b>circuit_key_material</b>. Return 0 on success else -1 on error. */
+STATIC int
+verify_establish_intro_cell(const trn_cell_establish_intro_t *cell,
+                            const uint8_t *circuit_key_material,
+                            size_t circuit_key_material_len)
+{
+  /* We only reach this function if the first byte of the cell is 0x02 which
+   * means that auth_key_type is of ed25519 type, hence this check should
+   * always pass. See hs_intro_received_establish_intro().  */
+  if (BUG(cell->auth_key_type != HS_INTRO_AUTH_KEY_TYPE_ED25519)) {
+    return -1;
+  }
+
+  /* Make sure the auth key length is of the right size for this type. For
+   * EXTRA safety, we check both the size of the array and the length which
+   * must be the same. Safety first!*/
+  if (trn_cell_establish_intro_getlen_auth_key(cell) != ED25519_PUBKEY_LEN ||
+      trn_cell_establish_intro_get_auth_key_len(cell) != ED25519_PUBKEY_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "ESTABLISH_INTRO auth key length is invalid");
+    return -1;
+  }
+
+  const uint8_t *msg = cell->start_cell;
+
+  /* Verify the sig */
+  {
+    ed25519_signature_t sig_struct;
+    const uint8_t *sig_array =
+      trn_cell_establish_intro_getconstarray_sig(cell);
+
+    /* Make sure the signature length is of the right size. For EXTRA safety,
+     * we check both the size of the array and the length which must be the
+     * same. Safety first!*/
+    if (trn_cell_establish_intro_getlen_sig(cell) != sizeof(sig_struct.sig) ||
+        trn_cell_establish_intro_get_sig_len(cell) != sizeof(sig_struct.sig)) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO sig len is invalid");
+      return -1;
+    }
+    /* We are now sure that sig_len is of the right size. */
+    memcpy(sig_struct.sig, sig_array, cell->sig_len);
+
+    ed25519_public_key_t auth_key;
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO, cell);
+
+    const size_t sig_msg_len = cell->end_sig_fields - msg;
+    int sig_mismatch = ed25519_checksig_prefixed(&sig_struct,
+                                                 msg, sig_msg_len,
+                                                 ESTABLISH_INTRO_SIG_PREFIX,
+                                                 &auth_key);
+    if (sig_mismatch) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO signature not as expected");
+      return -1;
+    }
+  }
+
+  /* Verify the MAC */
+  {
+    const size_t auth_msg_len = cell->end_mac_fields - msg;
+    uint8_t mac[DIGEST256_LEN];
+    crypto_mac_sha3_256(mac, sizeof(mac),
+                        circuit_key_material, circuit_key_material_len,
+                        msg, auth_msg_len);
+    if (tor_memneq(mac, cell->handshake_mac, sizeof(mac))) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO handshake_auth not as expected");
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/* Send an INTRO_ESTABLISHED cell to <b>circ</b>. */
+MOCK_IMPL(int,
+hs_intro_send_intro_established_cell,(or_circuit_t *circ))
+{
+  int ret;
+  uint8_t *encoded_cell = NULL;
+  ssize_t encoded_len, result_len;
+  trn_cell_intro_established_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(circ);
+
+  /* Build the cell payload. */
+  cell = trn_cell_intro_established_new();
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_intro_established_set_extensions(cell, ext);
+  /* Encode the cell to binary format. */
+  encoded_len = trn_cell_intro_established_encoded_len(cell);
+  tor_assert(encoded_len > 0);
+  encoded_cell = tor_malloc_zero(encoded_len);
+  result_len = trn_cell_intro_established_encode(encoded_cell, encoded_len,
+                                                cell);
+  tor_assert(encoded_len == result_len);
+
+  ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+                                     RELAY_COMMAND_INTRO_ESTABLISHED,
+                                     (char *) encoded_cell, encoded_len,
+                                     NULL);
+  /* On failure, the above function will close the circuit. */
+  trn_cell_intro_established_free(cell);
+  tor_free(encoded_cell);
+  return ret;
+}
+
+/** We received an ESTABLISH_INTRO <b>parsed_cell</b> on <b>circ</b>. It's
+ *  well-formed and passed our verifications. Perform appropriate actions to
+ *  establish an intro point. */
+static int
+handle_verified_establish_intro_cell(or_circuit_t *circ,
+                               const trn_cell_establish_intro_t *parsed_cell)
+{
+  /* Get the auth key of this intro point */
+  ed25519_public_key_t auth_key;
+  get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
+                         parsed_cell);
+
+  /* Then notify the hidden service that the intro point is established by
+     sending an INTRO_ESTABLISHED cell */
+  if (hs_intro_send_intro_established_cell(circ)) {
+    log_warn(LD_PROTOCOL, "Couldn't send INTRO_ESTABLISHED cell.");
+    return -1;
+  }
+
+  /* Associate intro point auth key with this circuit. */
+  hs_circuitmap_register_intro_circ_v3_relay_side(circ, &auth_key);
+  /* Repurpose this circuit into an intro circuit. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
+
+  return 0;
+}
+
+/** We just received an ESTABLISH_INTRO cell in <b>circ</b> with payload in
+ *  <b>request</b>. Handle it by making <b>circ</b> an intro circuit. Return 0
+ *  if everything went well, or -1 if there were errors. */
+static int
+handle_establish_intro(or_circuit_t *circ, const uint8_t *request,
+                       size_t request_len)
+{
+  int cell_ok, retval = -1;
+  trn_cell_establish_intro_t *parsed_cell = NULL;
+
+  tor_assert(circ);
+  tor_assert(request);
+
+  log_info(LD_REND, "Received an ESTABLISH_INTRO request on circuit %" PRIu32,
+           circ->p_circ_id);
+
+  /* Check that the circuit is in shape to become an intro point */
+  if (!hs_intro_circuit_is_suitable_for_establish_intro(circ)) {
+    goto err;
+  }
+
+  /* Parse the cell */
+  ssize_t parsing_result = trn_cell_establish_intro_parse(&parsed_cell,
+                                                         request, request_len);
+  if (parsing_result < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s ESTABLISH_INTRO cell.",
+           parsing_result == -1 ? "invalid" : "truncated");
+    goto err;
+  }
+
+  cell_ok = verify_establish_intro_cell(parsed_cell,
+                                        (uint8_t *) circ->rend_circ_nonce,
+                                        sizeof(circ->rend_circ_nonce));
+  if (cell_ok < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Failed to verify ESTABLISH_INTRO cell.");
+    goto err;
+  }
+
+  /* This cell is legit. Take the appropriate actions. */
+  cell_ok = handle_verified_establish_intro_cell(circ, parsed_cell);
+  if (cell_ok < 0) {
+    goto err;
+  }
+
+  /* We are done! */
+  retval = 0;
+  goto done;
+
+ err:
+  /* When sending the intro establish ack, on error the circuit can be marked
+   * as closed so avoid a double close. */
+  if (!TO_CIRCUIT(circ)->marked_for_close) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  }
+
+ done:
+  trn_cell_establish_intro_free(parsed_cell);
+  return retval;
+}
+
+/* Return True if circuit is suitable for being an intro circuit. */
+static int
+circuit_is_suitable_intro_point(const or_circuit_t *circ,
+                                const char *log_cell_type_str)
+{
+  tor_assert(circ);
+  tor_assert(log_cell_type_str);
+
+  /* Basic circuit state sanity checks. */
+  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s on non-OR circuit.", log_cell_type_str);
+    return 0;
+  }
+
+  if (circ->base_.n_chan) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s on non-edge circuit.", log_cell_type_str);
+    return 0;
+  }
+
+  /* Suitable. */
+  return 1;
+}
+
+/* Return True if circuit is suitable for being service-side intro circuit. */
+int
+hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ)
+{
+  return circuit_is_suitable_intro_point(circ, "ESTABLISH_INTRO");
+}
+
+/* We just received an ESTABLISH_INTRO cell in <b>circ</b>. Figure out of it's
+ * a legacy or a next gen cell, and pass it to the appropriate handler. */
+int
+hs_intro_received_establish_intro(or_circuit_t *circ, const uint8_t *request,
+                            size_t request_len)
+{
+  tor_assert(circ);
+  tor_assert(request);
+
+  if (request_len == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Empty ESTABLISH_INTRO cell.");
+    goto err;
+  }
+
+  /* Using the first byte of the cell, figure out the version of
+   * ESTABLISH_INTRO and pass it to the appropriate cell handler */
+  const uint8_t first_byte = request[0];
+  switch (first_byte) {
+    case HS_INTRO_AUTH_KEY_TYPE_LEGACY0:
+    case HS_INTRO_AUTH_KEY_TYPE_LEGACY1:
+      return rend_mid_establish_intro_legacy(circ, request, request_len);
+    case HS_INTRO_AUTH_KEY_TYPE_ED25519:
+      return handle_establish_intro(circ, request, request_len);
+    default:
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Unrecognized AUTH_KEY_TYPE %u.", first_byte);
+      goto err;
+  }
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Send an INTRODUCE_ACK cell onto the circuit <b>circ</b> with the status
+ * value in <b>status</b>. Depending on the status, it can be ACK or a NACK.
+ * Return 0 on success else a negative value on error which will close the
+ * circuit. */
+static int
+send_introduce_ack_cell(or_circuit_t *circ, hs_intro_ack_status_t status)
+{
+  int ret = -1;
+  uint8_t *encoded_cell = NULL;
+  ssize_t encoded_len, result_len;
+  trn_cell_introduce_ack_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(circ);
+
+  /* Setup the INTRODUCE_ACK cell. We have no extensions so the N_EXTENSIONS
+   * field is set to 0 by default with a new object. */
+  cell = trn_cell_introduce_ack_new();
+  ret = trn_cell_introduce_ack_set_status(cell, status);
+  /* We have no cell extensions in an INTRODUCE_ACK cell. */
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce_ack_set_extensions(cell, ext);
+  /* A wrong status is a very bad code flow error as this value is controlled
+   * by the code in this file and not an external input. This means we use a
+   * code that is not known by the trunnel ABI. */
+  tor_assert(ret == 0);
+  /* Encode the payload. We should never fail to get the encoded length. */
+  encoded_len = trn_cell_introduce_ack_encoded_len(cell);
+  tor_assert(encoded_len > 0);
+  encoded_cell = tor_malloc_zero(encoded_len);
+  result_len = trn_cell_introduce_ack_encode(encoded_cell, encoded_len, cell);
+  tor_assert(encoded_len == result_len);
+
+  ret = relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                     RELAY_COMMAND_INTRODUCE_ACK,
+                                     (char *) encoded_cell, encoded_len,
+                                     NULL);
+  /* On failure, the above function will close the circuit. */
+  trn_cell_introduce_ack_free(cell);
+  tor_free(encoded_cell);
+  return ret;
+}
+
+/* Validate a parsed INTRODUCE1 <b>cell</b>. Return 0 if valid or else a
+ * negative value for an invalid cell that should be NACKed. */
+STATIC int
+validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
+{
+  size_t legacy_key_id_len;
+  const uint8_t *legacy_key_id;
+
+  tor_assert(cell);
+
+  /* This code path SHOULD NEVER be reached if the cell is a legacy type so
+   * safety net here. The legacy ID must be zeroes in this case. */
+  legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell);
+  legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell);
+  if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
+    goto invalid;
+  }
+
+  /* The auth key of an INTRODUCE1 should be of type ed25519 thus leading to a
+   * known fixed length as well. */
+  if (trn_cell_introduce1_get_auth_key_type(cell) !=
+      HS_INTRO_AUTH_KEY_TYPE_ED25519) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell auth key type. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+  if (trn_cell_introduce1_get_auth_key_len(cell) != ED25519_PUBKEY_LEN ||
+      trn_cell_introduce1_getlen_auth_key(cell) != ED25519_PUBKEY_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell auth key length. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+  if (trn_cell_introduce1_getlen_encrypted(cell) == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell encrypted length. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* We just received a non legacy INTRODUCE1 cell on <b>client_circ</b> with
+ * the payload in <b>request</b> of size <b>request_len</b>. Return 0 if
+ * everything went well, or -1 if an error occurred. This function is in charge
+ * of sending back an INTRODUCE_ACK cell and will close client_circ on error.
+ */
+STATIC int
+handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
+                  size_t request_len)
+{
+  int ret = -1;
+  or_circuit_t *service_circ;
+  trn_cell_introduce1_t *parsed_cell;
+  hs_intro_ack_status_t status = HS_INTRO_ACK_STATUS_SUCCESS;
+
+  tor_assert(client_circ);
+  tor_assert(request);
+
+  /* Parse cell. Note that we can only parse the non encrypted section for
+   * which we'll use the authentication key to find the service introduction
+   * circuit and relay the cell on it. */
+  ssize_t cell_size = trn_cell_introduce1_parse(&parsed_cell, request,
+                                               request_len);
+  if (cell_size < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s INTRODUCE1 cell. Responding with NACK.",
+           cell_size == -1 ? "invalid" : "truncated");
+    /* Inform client that the INTRODUCE1 has a bad format. */
+    status = HS_INTRO_ACK_STATUS_BAD_FORMAT;
+    goto send_ack;
+  }
+
+  /* Once parsed validate the cell format. */
+  if (validate_introduce1_parsed_cell(parsed_cell) < 0) {
+    /* Inform client that the INTRODUCE1 has bad format. */
+    status = HS_INTRO_ACK_STATUS_BAD_FORMAT;
+    goto send_ack;
+  }
+
+  /* Find introduction circuit through our circuit map. */
+  {
+    ed25519_public_key_t auth_key;
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_INTRODUCE1, parsed_cell);
+    service_circ = hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    if (service_circ == NULL) {
+      char b64_key[ED25519_BASE64_LEN + 1];
+      ed25519_public_to_base64(b64_key, &auth_key);
+      log_info(LD_REND, "No intro circuit found for INTRODUCE1 cell "
+                        "with auth key %s from circuit %" PRIu32 ". "
+                        "Responding with NACK.",
+               safe_str(b64_key), client_circ->p_circ_id);
+      /* Inform the client that we don't know the requested service ID. */
+      status = HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+      goto send_ack;
+    }
+  }
+
+  /* Relay the cell to the service on its intro circuit with an INTRODUCE2
+   * cell which is the same exact payload. */
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(service_circ),
+                                   RELAY_COMMAND_INTRODUCE2,
+                                   (char *) request, request_len, NULL)) {
+    log_warn(LD_PROTOCOL, "Unable to send INTRODUCE2 cell to the service.");
+    /* Inform the client that we can't relay the cell. */
+    status = HS_INTRO_ACK_STATUS_CANT_RELAY;
+    goto send_ack;
+  }
+
+  /* Success! Send an INTRODUCE_ACK success status onto the client circuit. */
+  status = HS_INTRO_ACK_STATUS_SUCCESS;
+  ret = 0;
+
+ send_ack:
+  /* Send INTRODUCE_ACK or INTRODUCE_NACK to client */
+  if (send_introduce_ack_cell(client_circ, status) < 0) {
+    log_warn(LD_PROTOCOL, "Unable to send an INTRODUCE ACK status %d "
+                          "to client.", status);
+    /* Circuit has been closed on failure of transmission. */
+    goto done;
+  }
+  if (status != HS_INTRO_ACK_STATUS_SUCCESS) {
+    /* We just sent a NACK that is a non success status code so close the
+     * circuit because it's not useful to keep it open. Remember, a client can
+     * only send one INTRODUCE1 cell on a circuit. */
+    circuit_mark_for_close(TO_CIRCUIT(client_circ), END_CIRC_REASON_INTERNAL);
+  }
+ done:
+  trn_cell_introduce1_free(parsed_cell);
+  return ret;
+}
+
+/* Identify if the encoded cell we just received is a legacy one or not. The
+ * <b>request</b> should be at least DIGEST_LEN bytes long. */
+STATIC int
+introduce1_cell_is_legacy(const uint8_t *request)
+{
+  tor_assert(request);
+
+  /* If the first 20 bytes of the cell (DIGEST_LEN) are NOT zeroes, it
+   * indicates a legacy cell (v2). */
+  if (!tor_mem_is_zero((const char *) request, DIGEST_LEN)) {
+    /* Legacy cell. */
+    return 1;
+  }
+  /* Not a legacy cell. */
+  return 0;
+}
+
+/* Return true iff the circuit <b>circ</b> is suitable for receiving an
+ * INTRODUCE1 cell. */
+STATIC int
+circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Is this circuit an intro point circuit? */
+  if (!circuit_is_suitable_intro_point(circ, "INTRODUCE1")) {
+    return 0;
+  }
+
+  if (circ->already_received_introduce1) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Blocking multiple introductions on the same circuit. "
+           "Someone might be trying to attack a hidden service through "
+           "this relay.");
+    return 0;
+  }
+
+  return 1;
+}
+
+/* We just received an INTRODUCE1 cell on <b>circ</b>. Figure out which type
+ * it is and pass it to the appropriate handler. Return 0 on success else a
+ * negative value and the circuit is closed. */
+int
+hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
+                             size_t request_len)
+{
+  int ret;
+
+  tor_assert(circ);
+  tor_assert(request);
+
+  /* A cell that can't hold a DIGEST_LEN is invalid as we need to check if
+   * it's a legacy cell or not using the first DIGEST_LEN bytes. */
+  if (request_len < DIGEST_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Invalid INTRODUCE1 cell length.");
+    goto err;
+  }
+
+  /* Make sure we have a circuit that can have an INTRODUCE1 cell on it. */
+  if (!circuit_is_suitable_for_introduce1(circ)) {
+    /* We do not send a NACK because the circuit is not suitable for any kind
+     * of response or transmission as it's a violation of the protocol. */
+    goto err;
+  }
+  /* Mark the circuit that we got this cell. None are allowed after this as a
+   * DoS mitigation since one circuit with one client can hammer a service. */
+  circ->already_received_introduce1 = 1;
+
+  /* We are sure here to have at least DIGEST_LEN bytes. */
+  if (introduce1_cell_is_legacy(request)) {
+    /* Handle a legacy cell. */
+    ret = rend_mid_introduce_legacy(circ, request, request_len);
+  } else {
+    /* Handle a non legacy cell. */
+    ret = handle_introduce1(circ, request, request_len);
+  }
+  return ret;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Clear memory allocated by the given intropoint object ip (but don't free the
+ * object itself). */
+void
+hs_intropoint_clear(hs_intropoint_t *ip)
+{
+  if (ip == NULL) {
+    return;
+  }
+  tor_cert_free(ip->auth_key_cert);
+  SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *, ls,
+                    hs_desc_link_specifier_free(ls));
+  smartlist_free(ip->link_specifiers);
+  memset(ip, 0, sizeof(hs_intropoint_t));
+}
diff --git a/src/feature/hs/hs_intropoint.h b/src/feature/hs/hs_intropoint.h
new file mode 100644
index 0000000000..5f82920991
--- /dev/null
+++ b/src/feature/hs/hs_intropoint.h
@@ -0,0 +1,79 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_intropoint.h
+ * \brief Header file for hs_intropoint.c.
+ **/
+
+#ifndef TOR_HS_INTRO_H
+#define TOR_HS_INTRO_H
+
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "or/torcert.h"
+
+/* Authentication key type in an ESTABLISH_INTRO cell. */
+typedef enum {
+  HS_INTRO_AUTH_KEY_TYPE_LEGACY0 = 0x00,
+  HS_INTRO_AUTH_KEY_TYPE_LEGACY1 = 0x01,
+  HS_INTRO_AUTH_KEY_TYPE_ED25519 = 0x02,
+} hs_intro_auth_key_type_t;
+
+/* INTRODUCE_ACK status code. */
+typedef enum {
+  HS_INTRO_ACK_STATUS_SUCCESS    = 0x0000,
+  HS_INTRO_ACK_STATUS_UNKNOWN_ID = 0x0001,
+  HS_INTRO_ACK_STATUS_BAD_FORMAT = 0x0002,
+  HS_INTRO_ACK_STATUS_CANT_RELAY = 0x0003,
+} hs_intro_ack_status_t;
+
+/* Object containing introduction point common data between the service and
+ * the client side. */
+typedef struct hs_intropoint_t {
+  /* Does this intro point only supports legacy ID ?. */
+  unsigned int is_only_legacy : 1;
+
+  /* Authentication key certificate from the descriptor. */
+  tor_cert_t *auth_key_cert;
+  /* A list of link specifier. */
+  smartlist_t *link_specifiers;
+} hs_intropoint_t;
+
+int hs_intro_received_establish_intro(or_circuit_t *circ,
+                                      const uint8_t *request,
+                                      size_t request_len);
+int hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
+                                 size_t request_len);
+
+MOCK_DECL(int, hs_intro_send_intro_established_cell,(or_circuit_t *circ));
+
+/* also used by rendservice.c */
+int hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ);
+
+hs_intropoint_t *hs_intro_new(void);
+void hs_intropoint_clear(hs_intropoint_t *ip);
+
+#ifdef HS_INTROPOINT_PRIVATE
+
+#include "trunnel/hs/cell_establish_intro.h"
+#include "trunnel/hs/cell_introduce1.h"
+
+STATIC int
+verify_establish_intro_cell(const trn_cell_establish_intro_t *out,
+                            const uint8_t *circuit_key_material,
+                            size_t circuit_key_material_len);
+
+STATIC void
+get_auth_key_from_cell(ed25519_public_key_t *auth_key_out,
+                       unsigned int cell_type, const void *cell);
+
+STATIC int introduce1_cell_is_legacy(const uint8_t *request);
+STATIC int handle_introduce1(or_circuit_t *client_circ,
+                             const uint8_t *request, size_t request_len);
+STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell);
+STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ);
+
+#endif /* defined(HS_INTROPOINT_PRIVATE) */
+
+#endif /* !defined(TOR_HS_INTRO_H) */
+
diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c
new file mode 100644
index 0000000000..6cb01b57c9
--- /dev/null
+++ b/src/feature/hs/hs_service.c
@@ -0,0 +1,3644 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_service.c
+ * \brief Implement next generation hidden service functionality
+ **/
+
+#define HS_SERVICE_PRIVATE
+
+#include "or/or.h"
+#include "or/circpathbias.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/directory.h"
+#include "or/main.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/relay.h"
+#include "or/rendservice.h"
+#include "or/router.h"
+#include "or/routerkeys.h"
+#include "or/routerlist.h"
+#include "or/shared_random_client.h"
+#include "or/statefile.h"
+
+#include "or/hs_circuit.h"
+#include "or/hs_common.h"
+#include "or/hs_config.h"
+#include "or/hs_control.h"
+#include "or/hs_descriptor.h"
+#include "or/hs_ident.h"
+#include "or/hs_intropoint.h"
+#include "or/hs_service.h"
+#include "or/hs_stats.h"
+
+#include "or/dir_connection_st.h"
+#include "or/edge_connection_st.h"
+#include "or/extend_info_st.h"
+#include "or/networkstatus_st.h"
+#include "or/node_st.h"
+#include "or/origin_circuit_st.h"
+#include "or/or_state_st.h"
+#include "or/routerstatus_st.h"
+
+#include "lib/encoding/confline.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+/* Trunnel */
+#include "trunnel/ed25519_cert.h"
+#include "trunnel/hs/cell_common.h"
+#include "trunnel/hs/cell_establish_intro.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Helper macro. Iterate over every service in the global map. The var is the
+ * name of the service pointer. */
+#define FOR_EACH_SERVICE_BEGIN(var)                          \
+    STMT_BEGIN                                               \
+    hs_service_t **var##_iter, *var;                         \
+    HT_FOREACH(var##_iter, hs_service_ht, hs_service_map) {  \
+      var = *var##_iter;
+#define FOR_EACH_SERVICE_END } STMT_END ;
+
+/* Helper macro. Iterate over both current and previous descriptor of a
+ * service. The var is the name of the descriptor pointer. This macro skips
+ * any descriptor object of the service that is NULL. */
+#define FOR_EACH_DESCRIPTOR_BEGIN(service, var)                  \
+  STMT_BEGIN                                                     \
+    hs_service_descriptor_t *var;                                \
+    for (int var ## _loop_idx = 0; var ## _loop_idx < 2;         \
+         ++var ## _loop_idx) {                                   \
+      (var ## _loop_idx == 0) ? (var = service->desc_current) :  \
+                                (var = service->desc_next);      \
+      if (var == NULL) continue;
+#define FOR_EACH_DESCRIPTOR_END } STMT_END ;
+
+/* Onion service directory file names. */
+static const char fname_keyfile_prefix[] = "hs_ed25519";
+static const char fname_hostname[] = "hostname";
+static const char address_tld[] = "onion";
+
+/* Staging list of service object. When configuring service, we add them to
+ * this list considered a staging area and they will get added to our global
+ * map once the keys have been loaded. These two steps are separated because
+ * loading keys requires that we are an actual running tor process. */
+static smartlist_t *hs_service_staging_list;
+
+/** True if the list of available router descriptors might have changed which
+ *  might result in an altered hash ring. Check if the hash ring changed and
+ *  reupload if needed */
+static int consider_republishing_hs_descriptors = 0;
+
+/* Static declaration. */
+static void set_descriptor_revision_counter(hs_descriptor_t *hs_desc);
+static void move_descriptors(hs_service_t *src, hs_service_t *dst);
+
+/* Helper: Function to compare two objects in the service map. Return 1 if the
+ * two service have the same master public identity key. */
+static inline int
+hs_service_ht_eq(const hs_service_t *first, const hs_service_t *second)
+{
+  tor_assert(first);
+  tor_assert(second);
+  /* Simple key compare. */
+  return ed25519_pubkey_eq(&first->keys.identity_pk,
+                           &second->keys.identity_pk);
+}
+
+/* Helper: Function for the service hash table code below. The key used is the
+ * master public identity key which is ultimately the onion address. */
+static inline unsigned int
+hs_service_ht_hash(const hs_service_t *service)
+{
+  tor_assert(service);
+  return (unsigned int) siphash24g(service->keys.identity_pk.pubkey,
+                                   sizeof(service->keys.identity_pk.pubkey));
+}
+
+/* This is _the_ global hash map of hidden services which indexed the service
+ * contained in it by master public identity key which is roughly the onion
+ * address of the service. */
+static struct hs_service_ht *hs_service_map;
+
+/* Register the service hash table. */
+HT_PROTOTYPE(hs_service_ht,      /* Name of hashtable. */
+             hs_service_t,       /* Object contained in the map. */
+             hs_service_node,    /* The name of the HT_ENTRY member. */
+             hs_service_ht_hash, /* Hashing function. */
+             hs_service_ht_eq)   /* Compare function for objects. */
+
+HT_GENERATE2(hs_service_ht, hs_service_t, hs_service_node,
+             hs_service_ht_hash, hs_service_ht_eq,
+             0.6, tor_reallocarray, tor_free_)
+
+/* Query the given service map with a public key and return a service object
+ * if found else NULL. It is also possible to set a directory path in the
+ * search query. If pk is NULL, then it will be set to zero indicating the
+ * hash table to compare the directory path instead. */
+STATIC hs_service_t *
+find_service(hs_service_ht *map, const ed25519_public_key_t *pk)
+{
+  hs_service_t dummy_service;
+  tor_assert(map);
+  tor_assert(pk);
+  memset(&dummy_service, 0, sizeof(dummy_service));
+  ed25519_pubkey_copy(&dummy_service.keys.identity_pk, pk);
+  return HT_FIND(hs_service_ht, map, &dummy_service);
+}
+
+/* Register the given service in the given map. If the service already exists
+ * in the map, -1 is returned. On success, 0 is returned and the service
+ * ownership has been transferred to the global map. */
+STATIC int
+register_service(hs_service_ht *map, hs_service_t *service)
+{
+  tor_assert(map);
+  tor_assert(service);
+  tor_assert(!ed25519_public_key_is_zero(&service->keys.identity_pk));
+
+  if (find_service(map, &service->keys.identity_pk)) {
+    /* Existing service with the same key. Do not register it. */
+    return -1;
+  }
+  /* Taking ownership of the object at this point. */
+  HT_INSERT(hs_service_ht, map, service);
+
+  /* If we just modified the global map, we notify. */
+  if (map == hs_service_map) {
+    hs_service_map_has_changed();
+  }
+
+  return 0;
+}
+
+/* Remove a given service from the given map. If service is NULL or the
+ * service key is unset, return gracefully. */
+STATIC void
+remove_service(hs_service_ht *map, hs_service_t *service)
+{
+  hs_service_t *elm;
+
+  tor_assert(map);
+
+  /* Ignore if no service or key is zero. */
+  if (BUG(service == NULL) ||
+      BUG(ed25519_public_key_is_zero(&service->keys.identity_pk))) {
+    return;
+  }
+
+  elm = HT_REMOVE(hs_service_ht, map, service);
+  if (elm) {
+    tor_assert(elm == service);
+  } else {
+    log_warn(LD_BUG, "Could not find service in the global map "
+                     "while removing service %s",
+             escaped(service->config.directory_path));
+  }
+
+  /* If we just modified the global map, we notify. */
+  if (map == hs_service_map) {
+    hs_service_map_has_changed();
+  }
+}
+
+/* Set the default values for a service configuration object <b>c</b>. */
+static void
+set_service_default_config(hs_service_config_t *c,
+                           const or_options_t *options)
+{
+  (void) options;
+  tor_assert(c);
+  c->ports = smartlist_new();
+  c->directory_path = NULL;
+  c->max_streams_per_rdv_circuit = 0;
+  c->max_streams_close_circuit = 0;
+  c->num_intro_points = NUM_INTRO_POINTS_DEFAULT;
+  c->allow_unknown_ports = 0;
+  c->is_single_onion = 0;
+  c->dir_group_readable = 0;
+  c->is_ephemeral = 0;
+}
+
+/* From a service configuration object config, clear everything from it
+ * meaning free allocated pointers and reset the values. */
+static void
+service_clear_config(hs_service_config_t *config)
+{
+  if (config == NULL) {
+    return;
+  }
+  tor_free(config->directory_path);
+  if (config->ports) {
+    SMARTLIST_FOREACH(config->ports, rend_service_port_config_t *, p,
+                      rend_service_port_config_free(p););
+    smartlist_free(config->ports);
+  }
+  memset(config, 0, sizeof(*config));
+}
+
+/* Helper function to return a human readable description of the given intro
+ * point object.
+ *
+ * This function is not thread-safe. Each call to this invalidates the
+ * previous values returned by it. */
+static const char *
+describe_intro_point(const hs_service_intro_point_t *ip)
+{
+  /* Hex identity digest of the IP prefixed by the $ sign and ends with NUL
+   * byte hence the plus two. */
+  static char buf[HEX_DIGEST_LEN + 2];
+  const char *legacy_id = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                          const hs_desc_link_specifier_t *, lspec) {
+    if (lspec->type == LS_LEGACY_ID) {
+      legacy_id = (const char *) lspec->u.legacy_id;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(lspec);
+
+  /* For now, we only print the identity digest but we could improve this with
+   * much more information such as the ed25519 identity has well. */
+  buf[0] = '$';
+  if (legacy_id) {
+    base16_encode(buf + 1, HEX_DIGEST_LEN + 1, legacy_id, DIGEST_LEN);
+  }
+
+  return buf;
+}
+
+/* Return the lower bound of maximum INTRODUCE2 cells per circuit before we
+ * rotate intro point (defined by a consensus parameter or the default
+ * value). */
+static int32_t
+get_intro_point_min_introduce2(void)
+{
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_min_introduce2",
+                                 INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS,
+                                 0, INT32_MAX);
+}
+
+/* Return the upper bound of maximum INTRODUCE2 cells per circuit before we
+ * rotate intro point (defined by a consensus parameter or the default
+ * value). */
+static int32_t
+get_intro_point_max_introduce2(void)
+{
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_max_introduce2",
+                                 INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS,
+                                 0, INT32_MAX);
+}
+
+/* Return the minimum lifetime in seconds of an introduction point defined by a
+ * consensus parameter or the default value. */
+static int32_t
+get_intro_point_min_lifetime(void)
+{
+#define MIN_INTRO_POINT_LIFETIME_TESTING 10
+  if (get_options()->TestingTorNetwork) {
+    return MIN_INTRO_POINT_LIFETIME_TESTING;
+  }
+
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_min_lifetime",
+                                 INTRO_POINT_LIFETIME_MIN_SECONDS,
+                                 0, INT32_MAX);
+}
+
+/* Return the maximum lifetime in seconds of an introduction point defined by a
+ * consensus parameter or the default value. */
+static int32_t
+get_intro_point_max_lifetime(void)
+{
+#define MAX_INTRO_POINT_LIFETIME_TESTING 30
+  if (get_options()->TestingTorNetwork) {
+    return MAX_INTRO_POINT_LIFETIME_TESTING;
+  }
+
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_max_lifetime",
+                                 INTRO_POINT_LIFETIME_MAX_SECONDS,
+                                 0, INT32_MAX);
+}
+
+/* Return the number of extra introduction point defined by a consensus
+ * parameter or the default value. */
+static int32_t
+get_intro_point_num_extra(void)
+{
+  /* The [0, 128] range bounds the number of extra introduction point allowed.
+   * Above 128 intro points, it's getting a bit crazy. */
+  return networkstatus_get_param(NULL, "hs_intro_num_extra",
+                                 NUM_INTRO_POINTS_EXTRA, 0, 128);
+}
+
+/* Helper: Function that needs to return 1 for the HT for each loop which
+ * frees every service in an hash map. */
+static int
+ht_free_service_(struct hs_service_t *service, void *data)
+{
+  (void) data;
+  hs_service_free(service);
+  /* This function MUST return 1 so the given object is then removed from the
+   * service map leading to this free of the object being safe. */
+  return 1;
+}
+
+/* Free every service that can be found in the global map. Once done, clear
+ * and free the global map. */
+static void
+service_free_all(void)
+{
+  if (hs_service_map) {
+    /* The free helper function returns 1 so this is safe. */
+    hs_service_ht_HT_FOREACH_FN(hs_service_map, ht_free_service_, NULL);
+    HT_CLEAR(hs_service_ht, hs_service_map);
+    tor_free(hs_service_map);
+    hs_service_map = NULL;
+  }
+
+  if (hs_service_staging_list) {
+    /* Cleanup staging list. */
+    SMARTLIST_FOREACH(hs_service_staging_list, hs_service_t *, s,
+                      hs_service_free(s));
+    smartlist_free(hs_service_staging_list);
+    hs_service_staging_list = NULL;
+  }
+}
+
+/* Free a given service intro point object. */
+STATIC void
+service_intro_point_free_(hs_service_intro_point_t *ip)
+{
+  if (!ip) {
+    return;
+  }
+  memwipe(&ip->auth_key_kp, 0, sizeof(ip->auth_key_kp));
+  memwipe(&ip->enc_key_kp, 0, sizeof(ip->enc_key_kp));
+  crypto_pk_free(ip->legacy_key);
+  replaycache_free(ip->replay_cache);
+  hs_intropoint_clear(&ip->base);
+  tor_free(ip);
+}
+
+/* Helper: free an hs_service_intro_point_t object. This function is used by
+ * digest256map_free() which requires a void * pointer. */
+static void
+service_intro_point_free_void(void *obj)
+{
+  service_intro_point_free_(obj);
+}
+
+/* Return a newly allocated service intro point and fully initialized from the
+ * given extend_info_t ei if non NULL. If is_legacy is true, we also generate
+ * the legacy key. On error, NULL is returned.
+ *
+ * If ei is NULL, returns a hs_service_intro_point_t with an empty link
+ * specifier list and no onion key. (This is used for testing.)
+ *
+ * ei must be an extend_info_t containing an IPv4 address. (We will add supoort
+ * for IPv6 in a later release.) When calling extend_info_from_node(), pass
+ * 0 in for_direct_connection to make sure ei always has an IPv4 address. */
+STATIC hs_service_intro_point_t *
+service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy)
+{
+  hs_desc_link_specifier_t *ls;
+  hs_service_intro_point_t *ip;
+
+  ip = tor_malloc_zero(sizeof(*ip));
+  /* We'll create the key material. No need for extra strong, those are short
+   * term keys. */
+  ed25519_keypair_generate(&ip->auth_key_kp, 0);
+
+  { /* Set introduce2 max cells limit */
+    int32_t min_introduce2_cells = get_intro_point_min_introduce2();
+    int32_t max_introduce2_cells = get_intro_point_max_introduce2();
+    if (BUG(max_introduce2_cells < min_introduce2_cells)) {
+      goto err;
+    }
+    ip->introduce2_max = crypto_rand_int_range(min_introduce2_cells,
+                                               max_introduce2_cells);
+  }
+  { /* Set intro point lifetime */
+    int32_t intro_point_min_lifetime = get_intro_point_min_lifetime();
+    int32_t intro_point_max_lifetime = get_intro_point_max_lifetime();
+    if (BUG(intro_point_max_lifetime < intro_point_min_lifetime)) {
+      goto err;
+    }
+    ip->time_to_expire = time(NULL) +
+      crypto_rand_int_range(intro_point_min_lifetime,intro_point_max_lifetime);
+  }
+
+  ip->replay_cache = replaycache_new(0, 0);
+
+  /* Initialize the base object. We don't need the certificate object. */
+  ip->base.link_specifiers = smartlist_new();
+
+  /* Generate the encryption key for this intro point. */
+  curve25519_keypair_generate(&ip->enc_key_kp, 0);
+  /* Figure out if this chosen node supports v3 or is legacy only. */
+  if (is_legacy) {
+    ip->base.is_only_legacy = 1;
+    /* Legacy mode that is doesn't support v3+ with ed25519 auth key. */
+    ip->legacy_key = crypto_pk_new();
+    if (crypto_pk_generate_key(ip->legacy_key) < 0) {
+      goto err;
+    }
+    if (crypto_pk_get_digest(ip->legacy_key,
+                             (char *) ip->legacy_key_digest) < 0) {
+      goto err;
+    }
+  }
+
+  if (ei == NULL) {
+    goto done;
+  }
+
+  /* We'll try to add all link specifiers. Legacy is mandatory.
+   * IPv4 or IPv6 is required, and we always send IPv4. */
+  ls = hs_desc_link_specifier_new(ei, LS_IPV4);
+  /* It is impossible to have an extend info object without a v4. */
+  if (BUG(!ls)) {
+    goto err;
+  }
+  smartlist_add(ip->base.link_specifiers, ls);
+
+  ls = hs_desc_link_specifier_new(ei, LS_LEGACY_ID);
+  /* It is impossible to have an extend info object without an identity
+   * digest. */
+  if (BUG(!ls)) {
+    goto err;
+  }
+  smartlist_add(ip->base.link_specifiers, ls);
+
+  /* ed25519 identity key is optional for intro points */
+  ls = hs_desc_link_specifier_new(ei, LS_ED25519_ID);
+  if (ls) {
+    smartlist_add(ip->base.link_specifiers, ls);
+  }
+
+  /* IPv6 is not supported in this release. */
+
+  /* Finally, copy onion key from the extend_info_t object. */
+  memcpy(&ip->onion_key, &ei->curve25519_onion_key, sizeof(ip->onion_key));
+
+ done:
+  return ip;
+ err:
+  service_intro_point_free(ip);
+  return NULL;
+}
+
+/* Add the given intro point object to the given intro point map. The intro
+ * point MUST have its RSA encryption key set if this is a legacy type or the
+ * authentication key set otherwise. */
+STATIC void
+service_intro_point_add(digest256map_t *map, hs_service_intro_point_t *ip)
+{
+  hs_service_intro_point_t *old_ip_entry;
+
+  tor_assert(map);
+  tor_assert(ip);
+
+  old_ip_entry = digest256map_set(map, ip->auth_key_kp.pubkey.pubkey, ip);
+  /* Make sure we didn't just try to double-add an intro point */
+  tor_assert_nonfatal(!old_ip_entry);
+}
+
+/* For a given service, remove the intro point from that service's descriptors
+ * (check both current and next descriptor) */
+STATIC void
+service_intro_point_remove(const hs_service_t *service,
+                           const hs_service_intro_point_t *ip)
+{
+  tor_assert(service);
+  tor_assert(ip);
+
+  /* Trying all descriptors. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    /* We'll try to remove the descriptor on both descriptors which is not
+     * very expensive to do instead of doing loopup + remove. */
+    digest256map_remove(desc->intro_points.map,
+                        ip->auth_key_kp.pubkey.pubkey);
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* For a given service and authentication key, return the intro point or NULL
+ * if not found. This will check both descriptors in the service. */
+STATIC hs_service_intro_point_t *
+service_intro_point_find(const hs_service_t *service,
+                         const ed25519_public_key_t *auth_key)
+{
+  hs_service_intro_point_t *ip = NULL;
+
+  tor_assert(service);
+  tor_assert(auth_key);
+
+  /* Trying all descriptors to find the right intro point.
+   *
+   * Even if we use the same node as intro point in both descriptors, the node
+   * will have a different intro auth key for each descriptor since we generate
+   * a new one everytime we pick an intro point.
+   *
+   * After #22893 gets implemented, intro points will be moved to be
+   * per-service instead of per-descriptor so this function will need to
+   * change.
+   */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    if ((ip = digest256map_get(desc->intro_points.map,
+                               auth_key->pubkey)) != NULL) {
+      break;
+    }
+  } FOR_EACH_DESCRIPTOR_END;
+
+  return ip;
+}
+
+/* For a given service and intro point, return the descriptor for which the
+ * intro point is assigned to. NULL is returned if not found. */
+STATIC hs_service_descriptor_t *
+service_desc_find_by_intro(const hs_service_t *service,
+                           const hs_service_intro_point_t *ip)
+{
+  hs_service_descriptor_t *descp = NULL;
+
+  tor_assert(service);
+  tor_assert(ip);
+
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    if (digest256map_get(desc->intro_points.map,
+                         ip->auth_key_kp.pubkey.pubkey)) {
+      descp = desc;
+      break;
+    }
+  } FOR_EACH_DESCRIPTOR_END;
+
+  return descp;
+}
+
+/* From a circuit identifier, get all the possible objects associated with the
+ * ident. If not NULL, service, ip or desc are set if the object can be found.
+ * They are untouched if they can't be found.
+ *
+ * This is an helper function because we do those lookups often so it's more
+ * convenient to simply call this functions to get all the things at once. */
+STATIC void
+get_objects_from_ident(const hs_ident_circuit_t *ident,
+                       hs_service_t **service, hs_service_intro_point_t **ip,
+                       hs_service_descriptor_t **desc)
+{
+  hs_service_t *s;
+
+  tor_assert(ident);
+
+  /* Get service object from the circuit identifier. */
+  s = find_service(hs_service_map, &ident->identity_pk);
+  if (s && service) {
+    *service = s;
+  }
+
+  /* From the service object, get the intro point object of that circuit. The
+   * following will query both descriptors intro points list. */
+  if (s && ip) {
+    *ip = service_intro_point_find(s, &ident->intro_auth_pk);
+  }
+
+  /* Get the descriptor for this introduction point and service. */
+  if (s && ip && *ip && desc) {
+    *desc = service_desc_find_by_intro(s, *ip);
+  }
+}
+
+/* From a given intro point, return the first link specifier of type
+ * encountered in the link specifier list. Return NULL if it can't be found.
+ *
+ * The caller does NOT have ownership of the object, the intro point does. */
+static hs_desc_link_specifier_t *
+get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
+{
+  hs_desc_link_specifier_t *lnk_spec = NULL;
+
+  tor_assert(ip);
+
+  SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                          hs_desc_link_specifier_t *, ls) {
+    if (ls->type == type) {
+      lnk_spec = ls;
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+ end:
+  return lnk_spec;
+}
+
+/* Given a service intro point, return the node_t associated to it. This can
+ * return NULL if the given intro point has no legacy ID or if the node can't
+ * be found in the consensus. */
+STATIC const node_t *
+get_node_from_intro_point(const hs_service_intro_point_t *ip)
+{
+  const hs_desc_link_specifier_t *ls;
+
+  tor_assert(ip);
+
+  ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
+  if (BUG(!ls)) {
+    return NULL;
+  }
+  /* XXX In the future, we want to only use the ed25519 ID (#22173). */
+  return node_get_by_id((const char *) ls->u.legacy_id);
+}
+
+/* Given a service intro point, return the extend_info_t for it. This can
+ * return NULL if the node can't be found for the intro point or the extend
+ * info can't be created for the found node. If direct_conn is set, the extend
+ * info is validated on if we can connect directly. */
+static extend_info_t *
+get_extend_info_from_intro_point(const hs_service_intro_point_t *ip,
+                                 unsigned int direct_conn)
+{
+  extend_info_t *info = NULL;
+  const node_t *node;
+
+  tor_assert(ip);
+
+  node = get_node_from_intro_point(ip);
+  if (node == NULL) {
+    /* This can happen if the relay serving as intro point has been removed
+     * from the consensus. In that case, the intro point will be removed from
+     * the descriptor during the scheduled events. */
+    goto end;
+  }
+
+  /* In the case of a direct connection (single onion service), it is possible
+   * our firewall policy won't allow it so this can return a NULL value. */
+  info = extend_info_from_node(node, direct_conn);
+
+ end:
+  return info;
+}
+
+/* Return the number of introduction points that are established for the
+ * given descriptor. */
+static unsigned int
+count_desc_circuit_established(const hs_service_descriptor_t *desc)
+{
+  unsigned int count = 0;
+
+  tor_assert(desc);
+
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       const hs_service_intro_point_t *, ip) {
+    count += ip->circuit_established;
+  } DIGEST256MAP_FOREACH_END;
+
+  return count;
+}
+
+/* For a given service and descriptor of that service, close all active
+ * directory connections. */
+static void
+close_directory_connections(const hs_service_t *service,
+                            const hs_service_descriptor_t *desc)
+{
+  unsigned int count = 0;
+  smartlist_t *dir_conns;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* Close pending HS desc upload connections for the blinded key of 'desc'. */
+  dir_conns = connection_list_by_type_purpose(CONN_TYPE_DIR,
+                                              DIR_PURPOSE_UPLOAD_HSDESC);
+  SMARTLIST_FOREACH_BEGIN(dir_conns, connection_t *, conn) {
+    dir_connection_t *dir_conn = TO_DIR_CONN(conn);
+    if (ed25519_pubkey_eq(&dir_conn->hs_ident->identity_pk,
+                          &service->keys.identity_pk) &&
+        ed25519_pubkey_eq(&dir_conn->hs_ident->blinded_pk,
+                          &desc->blinded_kp.pubkey)) {
+      connection_mark_for_close(conn);
+      count++;
+      continue;
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  log_info(LD_REND, "Closed %u active service directory connections for "
+                    "descriptor %s of service %s",
+           count, safe_str_client(ed25519_fmt(&desc->blinded_kp.pubkey)),
+           safe_str_client(service->onion_address));
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(dir_conns);
+}
+
+/* Close all rendezvous circuits for the given service. */
+static void
+close_service_rp_circuits(hs_service_t *service)
+{
+  origin_circuit_t *ocirc = NULL;
+
+  tor_assert(service);
+
+  /* The reason we go over all circuit instead of using the circuitmap API is
+   * because most hidden service circuits are rendezvous circuits so there is
+   * no real improvement at getting all rendezvous circuits from the
+   * circuitmap and then going over them all to find the right ones.
+   * Furthermore, another option would have been to keep a list of RP cookies
+   * for a service but it creates an engineering complexity since we don't
+   * have a "RP circuit closed" event to clean it up properly so we avoid a
+   * memory DoS possibility. */
+
+  while ((ocirc = circuit_get_next_service_rp_circ(ocirc))) {
+    /* Only close circuits that are v3 and for this service. */
+    if (ocirc->hs_ident != NULL &&
+        ed25519_pubkey_eq(&ocirc->hs_ident->identity_pk,
+                          &service->keys.identity_pk)) {
+      /* Reason is FINISHED because service has been removed and thus the
+       * circuit is considered old/uneeded. When freed, it is removed from the
+       * hs circuitmap. */
+      circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+    }
+  }
+}
+
+/* Close the circuit(s) for the given map of introduction points. */
+static void
+close_intro_circuits(hs_service_intropoints_t *intro_points)
+{
+  tor_assert(intro_points);
+
+  DIGEST256MAP_FOREACH(intro_points->map, key,
+                       const hs_service_intro_point_t *, ip) {
+    origin_circuit_t *ocirc = hs_circ_service_get_intro_circ(ip);
+    if (ocirc) {
+      /* Reason is FINISHED because service has been removed and thus the
+       * circuit is considered old/uneeded. When freed, the circuit is removed
+       * from the HS circuitmap. */
+      circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+    }
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Close all introduction circuits for the given service. */
+static void
+close_service_intro_circuits(hs_service_t *service)
+{
+  tor_assert(service);
+
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    close_intro_circuits(&desc->intro_points);
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Close any circuits related to the given service. */
+static void
+close_service_circuits(hs_service_t *service)
+{
+  tor_assert(service);
+
+  /* Only support for version >= 3. */
+  if (BUG(service->config.version < HS_VERSION_THREE)) {
+    return;
+  }
+  /* Close intro points. */
+  close_service_intro_circuits(service);
+  /* Close rendezvous points. */
+  close_service_rp_circuits(service);
+}
+
+/* Move every ephemeral services from the src service map to the dst service
+ * map. It is possible that a service can't be register to the dst map which
+ * won't stop the process of moving them all but will trigger a log warn. */
+static void
+move_ephemeral_services(hs_service_ht *src, hs_service_ht *dst)
+{
+  hs_service_t **iter, **next;
+
+  tor_assert(src);
+  tor_assert(dst);
+
+  /* Iterate over the map to find ephemeral service and move them to the other
+   * map. We loop using this method to have a safe removal process. */
+  for (iter = HT_START(hs_service_ht, src); iter != NULL; iter = next) {
+    hs_service_t *s = *iter;
+    if (!s->config.is_ephemeral) {
+      /* Yeah, we are in a very manual loop :). */
+      next = HT_NEXT(hs_service_ht, src, iter);
+      continue;
+    }
+    /* Remove service from map and then register to it to the other map.
+     * Reminder that "*iter" and "s" are the same thing. */
+    next = HT_NEXT_RMV(hs_service_ht, src, iter);
+    if (register_service(dst, s) < 0) {
+      log_warn(LD_BUG, "Ephemeral service key is already being used. "
+                       "Skipping.");
+    }
+  }
+}
+
+/* Return a const string of the directory path escaped. If this is an
+ * ephemeral service, it returns "[EPHEMERAL]". This can only be called from
+ * the main thread because escaped() uses a static variable. */
+static const char *
+service_escaped_dir(const hs_service_t *s)
+{
+  return (s->config.is_ephemeral) ? "[EPHEMERAL]" :
+                                    escaped(s->config.directory_path);
+}
+
+/** Move the hidden service state from <b>src</b> to <b>dst</b>. We do this
+ *  when we receive a SIGHUP: <b>dst</b> is the post-HUP service */
+static void
+move_hs_state(hs_service_t *src_service, hs_service_t *dst_service)
+{
+  tor_assert(src_service);
+  tor_assert(dst_service);
+
+  hs_service_state_t *src = &src_service->state;
+  hs_service_state_t *dst = &dst_service->state;
+
+  /* Let's do a shallow copy */
+  dst->intro_circ_retry_started_time = src->intro_circ_retry_started_time;
+  dst->num_intro_circ_launched = src->num_intro_circ_launched;
+  /* Freeing a NULL replaycache triggers an info LD_BUG. */
+  if (dst->replay_cache_rend_cookie != NULL) {
+    replaycache_free(dst->replay_cache_rend_cookie);
+  }
+  dst->replay_cache_rend_cookie = src->replay_cache_rend_cookie;
+
+  src->replay_cache_rend_cookie = NULL; /* steal pointer reference */
+}
+
+/* Register services that are in the staging list. Once this function returns,
+ * the global service map will be set with the right content and all non
+ * surviving services will be cleaned up. */
+static void
+register_all_services(void)
+{
+  struct hs_service_ht *new_service_map;
+
+  tor_assert(hs_service_staging_list);
+
+  /* Allocate a new map that will replace the current one. */
+  new_service_map = tor_malloc_zero(sizeof(*new_service_map));
+  HT_INIT(hs_service_ht, new_service_map);
+
+  /* First step is to transfer all ephemeral services from the current global
+   * map to the new one we are constructing. We do not prune ephemeral
+   * services as the only way to kill them is by deleting it from the control
+   * port or stopping the tor daemon. */
+  move_ephemeral_services(hs_service_map, new_service_map);
+
+  SMARTLIST_FOREACH_BEGIN(hs_service_staging_list, hs_service_t *, snew) {
+    hs_service_t *s;
+
+    /* Check if that service is already in our global map and if so, we'll
+     * transfer the intro points to it. */
+    s = find_service(hs_service_map, &snew->keys.identity_pk);
+    if (s) {
+      /* Pass ownership of the descriptors from s (the current service) to
+       * snew (the newly configured one). */
+      move_descriptors(s, snew);
+      move_hs_state(s, snew);
+      /* Remove the service from the global map because after this, we need to
+       * go over the remaining service in that map that aren't surviving the
+       * reload to close their circuits. */
+      remove_service(hs_service_map, s);
+      hs_service_free(s);
+    }
+    /* Great, this service is now ready to be added to our new map. */
+    if (BUG(register_service(new_service_map, snew) < 0)) {
+      /* This should never happen because prior to registration, we validate
+       * every service against the entire set. Not being able to register a
+       * service means we failed to validate correctly. In that case, don't
+       * break tor and ignore the service but tell user. */
+      log_warn(LD_BUG, "Unable to register service with directory %s",
+               service_escaped_dir(snew));
+      SMARTLIST_DEL_CURRENT(hs_service_staging_list, snew);
+      hs_service_free(snew);
+    }
+  } SMARTLIST_FOREACH_END(snew);
+
+  /* Close any circuits associated with the non surviving services. Every
+   * service in the current global map are roaming. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    close_service_circuits(service);
+  } FOR_EACH_SERVICE_END;
+
+  /* Time to make the switch. We'll clear the staging list because its content
+   * has now changed ownership to the map. */
+  smartlist_clear(hs_service_staging_list);
+  service_free_all();
+  hs_service_map = new_service_map;
+  /* We've just register services into the new map and now we've replaced the
+   * global map with it so we have to notify that the change happened. When
+   * registering a service, the notify is only triggered if the destination
+   * map is the global map for which in here it was not. */
+  hs_service_map_has_changed();
+}
+
+/* Write the onion address of a given service to the given filename fname_ in
+ * the service directory. Return 0 on success else -1 on error. */
+STATIC int
+write_address_to_file(const hs_service_t *service, const char *fname_)
+{
+  int ret = -1;
+  char *fname = NULL;
+  char *addr_buf = NULL;
+
+  tor_assert(service);
+  tor_assert(fname_);
+
+  /* Construct the full address with the onion tld and write the hostname file
+   * to disk. */
+  tor_asprintf(&addr_buf, "%s.%s\n", service->onion_address, address_tld);
+  /* Notice here that we use the given "fname_". */
+  fname = hs_path_from_filename(service->config.directory_path, fname_);
+  if (write_str_to_file(fname, addr_buf, 0) < 0) {
+    log_warn(LD_REND, "Could not write onion address to hostname file %s",
+             escaped(fname));
+    goto end;
+  }
+
+#ifndef _WIN32
+  if (service->config.dir_group_readable) {
+    /* Mode to 0640. */
+    if (chmod(fname, S_IRUSR | S_IWUSR | S_IRGRP) < 0) {
+      log_warn(LD_FS, "Unable to make onion service hostname file %s "
+                      "group-readable.", escaped(fname));
+    }
+  }
+#endif /* !defined(_WIN32) */
+
+  /* Success. */
+  ret = 0;
+ end:
+  tor_free(fname);
+  tor_free(addr_buf);
+  return ret;
+}
+
+/* Load and/or generate private keys for the given service. On success, the
+ * hostname file will be written to disk along with the master private key iff
+ * the service is not configured for offline keys. Return 0 on success else -1
+ * on failure. */
+static int
+load_service_keys(hs_service_t *service)
+{
+  int ret = -1;
+  char *fname = NULL;
+  ed25519_keypair_t *kp;
+  const hs_service_config_t *config;
+
+  tor_assert(service);
+
+  config = &service->config;
+
+  /* Create and fix permission on service directory. We are about to write
+   * files to that directory so make sure it exists and has the right
+   * permissions. We do this here because at this stage we know that Tor is
+   * actually running and the service we have has been validated. */
+  if (BUG(hs_check_service_private_dir(get_options()->User,
+                                       config->directory_path,
+                                       config->dir_group_readable, 1) < 0)) {
+    goto end;
+  }
+
+  /* Try to load the keys from file or generate it if not found. */
+  fname = hs_path_from_filename(config->directory_path, fname_keyfile_prefix);
+  /* Don't ask for key creation, we want to know if we were able to load it or
+   * we had to generate it. Better logging! */
+  kp = ed_key_init_from_file(fname, INIT_ED_KEY_SPLIT, LOG_INFO, NULL, 0, 0,
+                             0, NULL);
+  if (!kp) {
+    log_info(LD_REND, "Unable to load keys from %s. Generating it...", fname);
+    /* We'll now try to generate the keys and for it we want the strongest
+     * randomness for it. The keypair will be written in different files. */
+    uint32_t key_flags = INIT_ED_KEY_CREATE | INIT_ED_KEY_EXTRA_STRONG |
+                         INIT_ED_KEY_SPLIT;
+    kp = ed_key_init_from_file(fname, key_flags, LOG_WARN, NULL, 0, 0, 0,
+                               NULL);
+    if (!kp) {
+      log_warn(LD_REND, "Unable to generate keys and save in %s.", fname);
+      goto end;
+    }
+  }
+
+  /* Copy loaded or generated keys to service object. */
+  ed25519_pubkey_copy(&service->keys.identity_pk, &kp->pubkey);
+  memcpy(&service->keys.identity_sk, &kp->seckey,
+         sizeof(service->keys.identity_sk));
+  /* This does a proper memory wipe. */
+  ed25519_keypair_free(kp);
+
+  /* Build onion address from the newly loaded keys. */
+  tor_assert(service->config.version <= UINT8_MAX);
+  hs_build_address(&service->keys.identity_pk,
+                   (uint8_t) service->config.version,
+                   service->onion_address);
+
+  /* Write onion address to hostname file. */
+  if (write_address_to_file(service, fname_hostname) < 0) {
+    goto end;
+  }
+
+  /* Succes. */
+  ret = 0;
+ end:
+  tor_free(fname);
+  return ret;
+}
+
+/* Free a given service descriptor object and all key material is wiped. */
+STATIC void
+service_descriptor_free_(hs_service_descriptor_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+  hs_descriptor_free(desc->desc);
+  memwipe(&desc->signing_kp, 0, sizeof(desc->signing_kp));
+  memwipe(&desc->blinded_kp, 0, sizeof(desc->blinded_kp));
+  /* Cleanup all intro points. */
+  digest256map_free(desc->intro_points.map, service_intro_point_free_void);
+  digestmap_free(desc->intro_points.failed_id, tor_free_);
+  if (desc->previous_hsdirs) {
+    SMARTLIST_FOREACH(desc->previous_hsdirs, char *, s, tor_free(s));
+    smartlist_free(desc->previous_hsdirs);
+  }
+  tor_free(desc);
+}
+
+/* Return a newly allocated service descriptor object. */
+STATIC hs_service_descriptor_t *
+service_descriptor_new(void)
+{
+  hs_service_descriptor_t *sdesc = tor_malloc_zero(sizeof(*sdesc));
+  sdesc->desc = tor_malloc_zero(sizeof(hs_descriptor_t));
+  /* Initialize the intro points map. */
+  sdesc->intro_points.map = digest256map_new();
+  sdesc->intro_points.failed_id = digestmap_new();
+  sdesc->previous_hsdirs = smartlist_new();
+  return sdesc;
+}
+
+/* Move descriptor(s) from the src service to the dst service. We do this
+ * during SIGHUP when we re-create our hidden services. */
+static void
+move_descriptors(hs_service_t *src, hs_service_t *dst)
+{
+  tor_assert(src);
+  tor_assert(dst);
+
+  if (src->desc_current) {
+    /* Nothing should be there, but clean it up just in case */
+    if (BUG(dst->desc_current)) {
+      service_descriptor_free(dst->desc_current);
+    }
+    dst->desc_current = src->desc_current;
+    src->desc_current = NULL;
+  }
+
+  if (src->desc_next) {
+    /* Nothing should be there, but clean it up just in case */
+    if (BUG(dst->desc_next)) {
+      service_descriptor_free(dst->desc_next);
+    }
+    dst->desc_next = src->desc_next;
+    src->desc_next = NULL;
+  }
+}
+
+/* From the given service, remove all expired failing intro points for each
+ * descriptor. */
+static void
+remove_expired_failing_intro(hs_service_t *service, time_t now)
+{
+  tor_assert(service);
+
+  /* For both descriptors, cleanup the failing intro points list. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    DIGESTMAP_FOREACH_MODIFY(desc->intro_points.failed_id, key, time_t *, t) {
+      time_t failure_time = *t;
+      if ((failure_time + INTRO_CIRC_RETRY_PERIOD) <= now) {
+        MAP_DEL_CURRENT(key);
+        tor_free(t);
+      }
+    } DIGESTMAP_FOREACH_END;
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* For the given descriptor desc, put all node_t object found from its failing
+ * intro point list and put them in the given node_list. */
+static void
+setup_intro_point_exclude_list(const hs_service_descriptor_t *desc,
+                               smartlist_t *node_list)
+{
+  tor_assert(desc);
+  tor_assert(node_list);
+
+  DIGESTMAP_FOREACH(desc->intro_points.failed_id, key, time_t *, t) {
+    (void) t; /* Make gcc happy. */
+    const node_t *node = node_get_by_id(key);
+    if (node) {
+      smartlist_add(node_list, (void *) node);
+    }
+  } DIGESTMAP_FOREACH_END;
+}
+
+/* For the given failing intro point ip, we add its time of failure to the
+ * failed map and index it by identity digest (legacy ID) in the descriptor
+ * desc failed id map. */
+static void
+remember_failing_intro_point(const hs_service_intro_point_t *ip,
+                             hs_service_descriptor_t *desc, time_t now)
+{
+  time_t *time_of_failure, *prev_ptr;
+  const hs_desc_link_specifier_t *legacy_ls;
+
+  tor_assert(ip);
+  tor_assert(desc);
+
+  time_of_failure = tor_malloc_zero(sizeof(time_t));
+  *time_of_failure = now;
+  legacy_ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
+  tor_assert(legacy_ls);
+  prev_ptr = digestmap_set(desc->intro_points.failed_id,
+                           (const char *) legacy_ls->u.legacy_id,
+                           time_of_failure);
+  tor_free(prev_ptr);
+}
+
+/* Copy the descriptor link specifier object from src to dst. */
+static void
+link_specifier_copy(hs_desc_link_specifier_t *dst,
+                    const hs_desc_link_specifier_t *src)
+{
+  tor_assert(dst);
+  tor_assert(src);
+  memcpy(dst, src, sizeof(hs_desc_link_specifier_t));
+}
+
+/* Using a given descriptor signing keypair signing_kp, a service intro point
+ * object ip and the time now, setup the content of an already allocated
+ * descriptor intro desc_ip.
+ *
+ * Return 0 on success else a negative value. */
+static int
+setup_desc_intro_point(const ed25519_keypair_t *signing_kp,
+                       const hs_service_intro_point_t *ip,
+                       time_t now, hs_desc_intro_point_t *desc_ip)
+{
+  int ret = -1;
+  time_t nearest_hour = now - (now % 3600);
+
+  tor_assert(signing_kp);
+  tor_assert(ip);
+  tor_assert(desc_ip);
+
+  /* Copy the onion key. */
+  memcpy(&desc_ip->onion_key, &ip->onion_key, sizeof(desc_ip->onion_key));
+
+  /* Key and certificate material. */
+  desc_ip->auth_key_cert = tor_cert_create(signing_kp,
+                                           CERT_TYPE_AUTH_HS_IP_KEY,
+                                           &ip->auth_key_kp.pubkey,
+                                           nearest_hour,
+                                           HS_DESC_CERT_LIFETIME,
+                                           CERT_FLAG_INCLUDE_SIGNING_KEY);
+  if (desc_ip->auth_key_cert == NULL) {
+    log_warn(LD_REND, "Unable to create intro point auth-key certificate");
+    goto done;
+  }
+
+  /* Copy link specifier(s). */
+  SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                          const hs_desc_link_specifier_t *, ls) {
+    hs_desc_link_specifier_t *copy = tor_malloc_zero(sizeof(*copy));
+    link_specifier_copy(copy, ls);
+    smartlist_add(desc_ip->link_specifiers, copy);
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* For a legacy intro point, we'll use an RSA/ed cross certificate. */
+  if (ip->base.is_only_legacy) {
+    desc_ip->legacy.key = crypto_pk_dup_key(ip->legacy_key);
+    /* Create cross certification cert. */
+    ssize_t cert_len = tor_make_rsa_ed25519_crosscert(
+                                    &signing_kp->pubkey,
+                                    desc_ip->legacy.key,
+                                    nearest_hour + HS_DESC_CERT_LIFETIME,
+                                    &desc_ip->legacy.cert.encoded);
+    if (cert_len < 0) {
+      log_warn(LD_REND, "Unable to create enc key legacy cross cert.");
+      goto done;
+    }
+    desc_ip->legacy.cert.len = cert_len;
+  }
+
+  /* Encryption key and its cross certificate. */
+  {
+    ed25519_public_key_t ed25519_pubkey;
+
+    /* Use the public curve25519 key. */
+    memcpy(&desc_ip->enc_key, &ip->enc_key_kp.pubkey,
+           sizeof(desc_ip->enc_key));
+    /* The following can't fail. */
+    ed25519_public_key_from_curve25519_public_key(&ed25519_pubkey,
+                                                  &ip->enc_key_kp.pubkey,
+                                                  0);
+    desc_ip->enc_key_cert = tor_cert_create(signing_kp,
+                                            CERT_TYPE_CROSS_HS_IP_KEYS,
+                                            &ed25519_pubkey, nearest_hour,
+                                            HS_DESC_CERT_LIFETIME,
+                                            CERT_FLAG_INCLUDE_SIGNING_KEY);
+    if (desc_ip->enc_key_cert == NULL) {
+      log_warn(LD_REND, "Unable to create enc key curve25519 cross cert.");
+      goto done;
+    }
+  }
+  /* Success. */
+  ret = 0;
+
+ done:
+  return ret;
+}
+
+/* Using the given descriptor from the given service, build the descriptor
+ * intro point list so we can then encode the descriptor for publication. This
+ * function does not pick intro points, they have to be in the descriptor
+ * current map. Cryptographic material (keys) must be initialized in the
+ * descriptor for this function to make sense. */
+static void
+build_desc_intro_points(const hs_service_t *service,
+                        hs_service_descriptor_t *desc, time_t now)
+{
+  hs_desc_encrypted_data_t *encrypted;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* Ease our life. */
+  encrypted = &desc->desc->encrypted_data;
+  /* Cleanup intro points, we are about to set them from scratch. */
+  hs_descriptor_clear_intro_points(desc->desc);
+
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       const hs_service_intro_point_t *, ip) {
+    hs_desc_intro_point_t *desc_ip = hs_desc_intro_point_new();
+    if (setup_desc_intro_point(&desc->signing_kp, ip, now, desc_ip) < 0) {
+      hs_desc_intro_point_free(desc_ip);
+      continue;
+    }
+    /* We have a valid descriptor intro point. Add it to the list. */
+    smartlist_add(encrypted->intro_points, desc_ip);
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Populate the descriptor encrypted section from the given service object.
+ * This will generate a valid list of introduction points that can be used
+ * after for circuit creation. Return 0 on success else -1 on error. */
+static int
+build_service_desc_encrypted(const hs_service_t *service,
+                             hs_service_descriptor_t *desc)
+{
+  hs_desc_encrypted_data_t *encrypted;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  encrypted = &desc->desc->encrypted_data;
+
+  encrypted->create2_ntor = 1;
+  encrypted->single_onion_service = service->config.is_single_onion;
+
+  /* Setup introduction points from what we have in the service. */
+  if (encrypted->intro_points == NULL) {
+    encrypted->intro_points = smartlist_new();
+  }
+  /* We do NOT build introduction point yet, we only do that once the circuit
+   * have been opened. Until we have the right number of introduction points,
+   * we do not encode anything in the descriptor. */
+
+  /* XXX: Support client authorization (#20700). */
+  encrypted->intro_auth_types = NULL;
+  return 0;
+}
+
+/* Populate the descriptor plaintext section from the given service object.
+ * The caller must make sure that the keys in the descriptors are valid that
+ * is are non-zero. Return 0 on success else -1 on error. */
+static int
+build_service_desc_plaintext(const hs_service_t *service,
+                             hs_service_descriptor_t *desc, time_t now)
+{
+  int ret = -1;
+  hs_desc_plaintext_data_t *plaintext;
+
+  tor_assert(service);
+  tor_assert(desc);
+  /* XXX: Use a "assert_desc_ok()" ? */
+  tor_assert(!tor_mem_is_zero((char *) &desc->blinded_kp,
+                              sizeof(desc->blinded_kp)));
+  tor_assert(!tor_mem_is_zero((char *) &desc->signing_kp,
+                              sizeof(desc->signing_kp)));
+
+  /* Set the subcredential. */
+  hs_get_subcredential(&service->keys.identity_pk, &desc->blinded_kp.pubkey,
+                       desc->desc->subcredential);
+
+  plaintext = &desc->desc->plaintext_data;
+
+  plaintext->version = service->config.version;
+  plaintext->lifetime_sec = HS_DESC_DEFAULT_LIFETIME;
+  plaintext->signing_key_cert =
+    tor_cert_create(&desc->blinded_kp, CERT_TYPE_SIGNING_HS_DESC,
+                    &desc->signing_kp.pubkey, now, HS_DESC_CERT_LIFETIME,
+                    CERT_FLAG_INCLUDE_SIGNING_KEY);
+  if (plaintext->signing_key_cert == NULL) {
+    log_warn(LD_REND, "Unable to create descriptor signing certificate for "
+                      "service %s",
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+  /* Copy public key material to go in the descriptor. */
+  ed25519_pubkey_copy(&plaintext->signing_pubkey, &desc->signing_kp.pubkey);
+  ed25519_pubkey_copy(&plaintext->blinded_pubkey, &desc->blinded_kp.pubkey);
+  /* Success. */
+  ret = 0;
+
+ end:
+  return ret;
+}
+
+/* For the given service and descriptor object, create the key material which
+ * is the blinded keypair and the descriptor signing keypair. Return 0 on
+ * success else -1 on error where the generated keys MUST be ignored. */
+static int
+build_service_desc_keys(const hs_service_t *service,
+                        hs_service_descriptor_t *desc,
+                        uint64_t time_period_num)
+{
+  int ret = 0;
+  ed25519_keypair_t kp;
+
+  tor_assert(desc);
+  tor_assert(!tor_mem_is_zero((char *) &service->keys.identity_pk,
+             ED25519_PUBKEY_LEN));
+
+  /* XXX: Support offline key feature (#18098). */
+
+  /* Copy the identity keys to the keypair so we can use it to create the
+   * blinded key. */
+  memcpy(&kp.pubkey, &service->keys.identity_pk, sizeof(kp.pubkey));
+  memcpy(&kp.seckey, &service->keys.identity_sk, sizeof(kp.seckey));
+  /* Build blinded keypair for this time period. */
+  hs_build_blinded_keypair(&kp, NULL, 0, time_period_num, &desc->blinded_kp);
+  /* Let's not keep too much traces of our keys in memory. */
+  memwipe(&kp, 0, sizeof(kp));
+
+  /* No need for extra strong, this is a temporary key only for this
+   * descriptor. Nothing long term. */
+  if (ed25519_keypair_generate(&desc->signing_kp, 0) < 0) {
+    log_warn(LD_REND, "Can't generate descriptor signing keypair for "
+                      "service %s",
+             safe_str_client(service->onion_address));
+    ret = -1;
+  }
+
+  return ret;
+}
+
+/* Given a service and the current time, build a descriptor for the service.
+ * This function does not pick introduction point, this needs to be done by
+ * the update function. On success, desc_out will point to the newly allocated
+ * descriptor object.
+ *
+ * This can error if we are unable to create keys or certificate. */
+static void
+build_service_descriptor(hs_service_t *service, time_t now,
+                         uint64_t time_period_num,
+                         hs_service_descriptor_t **desc_out)
+{
+  char *encoded_desc;
+  hs_service_descriptor_t *desc;
+
+  tor_assert(service);
+  tor_assert(desc_out);
+
+  desc = service_descriptor_new();
+  desc->time_period_num = time_period_num;
+
+  /* Create the needed keys so we can setup the descriptor content. */
+  if (build_service_desc_keys(service, desc, time_period_num) < 0) {
+    goto err;
+  }
+  /* Setup plaintext descriptor content. */
+  if (build_service_desc_plaintext(service, desc, now) < 0) {
+    goto err;
+  }
+  /* Setup encrypted descriptor content. */
+  if (build_service_desc_encrypted(service, desc) < 0) {
+    goto err;
+  }
+
+  /* Set the revision counter for this descriptor */
+  set_descriptor_revision_counter(desc->desc);
+
+  /* Let's make sure that we've created a descriptor that can actually be
+   * encoded properly. This function also checks if the encoded output is
+   * decodable after. */
+  if (BUG(hs_desc_encode_descriptor(desc->desc, &desc->signing_kp,
+                                    &encoded_desc) < 0)) {
+    goto err;
+  }
+  tor_free(encoded_desc);
+
+  /* Assign newly built descriptor to the next slot. */
+  *desc_out = desc;
+  /* Fire a CREATED control port event. */
+  hs_control_desc_event_created(service->onion_address,
+                                &desc->blinded_kp.pubkey);
+  return;
+
+ err:
+  service_descriptor_free(desc);
+}
+
+/* Build both descriptors for the given service that has just booted up.
+ * Because it's a special case, it deserves its special function ;). */
+static void
+build_descriptors_for_new_service(hs_service_t *service, time_t now)
+{
+  uint64_t current_desc_tp, next_desc_tp;
+
+  tor_assert(service);
+  /* These are the conditions for a new service. */
+  tor_assert(!service->desc_current);
+  tor_assert(!service->desc_next);
+
+  /*
+   * +------------------------------------------------------------------+
+   * |                                                                  |
+   * | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   * | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   * |                                                                  |
+   * |  $==========|-----------$===========|-----------$===========|    |
+   * |                             ^         ^                          |
+   * |                             A         B                          |
+   * +------------------------------------------------------------------+
+   *
+   * Case A: The service boots up before a new time period, the current time
+   * period is thus TP#1 and the next is TP#2 which for both we have access to
+   * their SRVs.
+   *
+   * Case B: The service boots up inside TP#2, we can't use the TP#3 for the
+   * next descriptor because we don't have the SRV#3 so the current should be
+   * TP#1 and next TP#2.
+   */
+
+  if (hs_in_period_between_tp_and_srv(NULL, now)) {
+    /* Case B from the above, inside of the new time period. */
+    current_desc_tp = hs_get_previous_time_period_num(0); /* TP#1 */
+    next_desc_tp = hs_get_time_period_num(0);             /* TP#2 */
+  } else {
+    /* Case A from the above, outside of the new time period. */
+    current_desc_tp = hs_get_time_period_num(0);    /* TP#1 */
+    next_desc_tp = hs_get_next_time_period_num(0);  /* TP#2 */
+  }
+
+  /* Build descriptors. */
+  build_service_descriptor(service, now, current_desc_tp,
+                           &service->desc_current);
+  build_service_descriptor(service, now, next_desc_tp,
+                           &service->desc_next);
+  log_info(LD_REND, "Hidden service %s has just started. Both descriptors "
+                    "built. Now scheduled for upload.",
+           safe_str_client(service->onion_address));
+}
+
+/* Build descriptors for each service if needed. There are conditions to build
+ * a descriptor which are details in the function. */
+STATIC void
+build_all_descriptors(time_t now)
+{
+  FOR_EACH_SERVICE_BEGIN(service) {
+
+    /* A service booting up will have both descriptors to NULL. No other cases
+     * makes both descriptor non existent. */
+    if (service->desc_current == NULL && service->desc_next == NULL) {
+      build_descriptors_for_new_service(service, now);
+      continue;
+    }
+
+    /* Reaching this point means we are pass bootup so at runtime. We should
+     * *never* have an empty current descriptor. If the next descriptor is
+     * empty, we'll try to build it for the next time period. This only
+     * happens when we rotate meaning that we are guaranteed to have a new SRV
+     * at that point for the next time period. */
+    if (BUG(service->desc_current == NULL)) {
+      continue;
+    }
+
+    if (service->desc_next == NULL) {
+      build_service_descriptor(service, now, hs_get_next_time_period_num(0),
+                               &service->desc_next);
+      log_info(LD_REND, "Hidden service %s next descriptor successfully "
+                        "built. Now scheduled for upload.",
+               safe_str_client(service->onion_address));
+    }
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Randomly pick a node to become an introduction point but not present in the
+ * given exclude_nodes list. The chosen node is put in the exclude list
+ * regardless of success or not because in case of failure, the node is simply
+ * unsusable from that point on.
+ *
+ * If direct_conn is set, try to pick a node that our local firewall/policy
+ * allows us to connect to directly. If we can't find any, return NULL.
+ * This function supports selecting dual-stack nodes for direct single onion
+ * service IPv6 connections. But it does not send IPv6 addresses in link
+ * specifiers. (Current clients don't use IPv6 addresses to extend, and
+ * direct client connections to intro points are not supported.)
+ *
+ * Return a newly allocated service intro point ready to be used for encoding.
+ * Return NULL on error. */
+static hs_service_intro_point_t *
+pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
+{
+  const node_t *node;
+  extend_info_t *info = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  /* Normal 3-hop introduction point flags. */
+  router_crn_flags_t flags = CRN_NEED_UPTIME | CRN_NEED_DESC;
+  /* Single onion flags. */
+  router_crn_flags_t direct_flags = flags | CRN_PREF_ADDR | CRN_DIRECT_CONN;
+
+  node = router_choose_random_node(exclude_nodes, get_options()->ExcludeNodes,
+                                   direct_conn ? direct_flags : flags);
+  /* Unable to find a node. When looking for a node for a direct connection,
+   * we could try a 3-hop path instead. We'll add support for this in a later
+   * release. */
+  if (!node) {
+    goto err;
+  }
+
+  /* We have a suitable node, add it to the exclude list. We do this *before*
+   * we can validate the extend information because even in case of failure,
+   * we don't want to use that node anymore. */
+  smartlist_add(exclude_nodes, (void *) node);
+
+  /* We do this to ease our life but also this call makes appropriate checks
+   * of the node object such as validating ntor support for instance.
+   *
+   * We must provide an extend_info for clients to connect over a 3-hop path,
+   * so we don't pass direct_conn here. */
+  info = extend_info_from_node(node, 0);
+  if (BUG(info == NULL)) {
+    goto err;
+  }
+
+  /* Let's do a basic sanity check here so that we don't end up advertising the
+   * ed25519 identity key of relays that don't actually support the link
+   * protocol */
+  if (!node_supports_ed25519_link_authentication(node, 0)) {
+    tor_assert_nonfatal(ed25519_public_key_is_zero(&info->ed_identity));
+  } else {
+    /* Make sure we *do* have an ed key if we support the link authentication.
+     * Sending an empty key would result in a failure to extend. */
+    tor_assert_nonfatal(!ed25519_public_key_is_zero(&info->ed_identity));
+  }
+
+  /* Create our objects and populate them with the node information. */
+  ip = service_intro_point_new(info, !node_supports_ed25519_hs_intro(node));
+  if (ip == NULL) {
+    goto err;
+  }
+
+  log_info(LD_REND, "Picked intro point: %s", extend_info_describe(info));
+  extend_info_free(info);
+  return ip;
+ err:
+  service_intro_point_free(ip);
+  extend_info_free(info);
+  return NULL;
+}
+
+/* For a given descriptor from the given service, pick any needed intro points
+ * and update the current map with those newly picked intro points. Return the
+ * number node that might have been added to the descriptor current map. */
+static unsigned int
+pick_needed_intro_points(hs_service_t *service,
+                         hs_service_descriptor_t *desc)
+{
+  int i = 0, num_needed_ip;
+  smartlist_t *exclude_nodes = smartlist_new();
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* Compute how many intro points we actually need to open. */
+  num_needed_ip = service->config.num_intro_points -
+                  digest256map_size(desc->intro_points.map);
+  if (BUG(num_needed_ip < 0)) {
+    /* Let's not make tor freak out here and just skip this. */
+    goto done;
+  }
+
+  /* We want to end up with config.num_intro_points intro points, but if we
+   * have no intro points at all (chances are they all cycled or we are
+   * starting up), we launch get_intro_point_num_extra() extra circuits and
+   * use the first config.num_intro_points that complete. See proposal #155,
+   * section 4 for the rationale of this which is purely for performance.
+   *
+   * The ones after the first config.num_intro_points will be converted to
+   * 'General' internal circuits and then we'll drop them from the list of
+   * intro points. */
+  if (digest256map_size(desc->intro_points.map) == 0) {
+    num_needed_ip += get_intro_point_num_extra();
+  }
+
+  /* Build an exclude list of nodes of our intro point(s). The expiring intro
+   * points are OK to pick again because this is afterall a concept of round
+   * robin so they are considered valid nodes to pick again. */
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       hs_service_intro_point_t *, ip) {
+    const node_t *intro_node = get_node_from_intro_point(ip);
+    if (intro_node) {
+      smartlist_add(exclude_nodes, (void*)intro_node);
+    }
+  } DIGEST256MAP_FOREACH_END;
+  /* Also, add the failing intro points that our descriptor encounteered in
+   * the exclude node list. */
+  setup_intro_point_exclude_list(desc, exclude_nodes);
+
+  for (i = 0; i < num_needed_ip; i++) {
+    hs_service_intro_point_t *ip;
+
+    /* This function will add the picked intro point node to the exclude nodes
+     * list so we don't pick the same one at the next iteration. */
+    ip = pick_intro_point(service->config.is_single_onion, exclude_nodes);
+    if (ip == NULL) {
+      /* If we end up unable to pick an introduction point it is because we
+       * can't find suitable node and calling this again is highly unlikely to
+       * give us a valid node all of the sudden. */
+      log_info(LD_REND, "Unable to find a suitable node to be an "
+                        "introduction point for service %s.",
+               safe_str_client(service->onion_address));
+      goto done;
+    }
+    /* Valid intro point object, add it to the descriptor current map. */
+    service_intro_point_add(desc->intro_points.map, ip);
+  }
+  /* We've successfully picked all our needed intro points thus none are
+   * missing which will tell our upload process to expect the number of
+   * circuits to be the number of configured intro points circuits and not the
+   * number of intro points object that we have. */
+  desc->missing_intro_points = 0;
+
+  /* Success. */
+ done:
+  /* We don't have ownership of the node_t object in this list. */
+  smartlist_free(exclude_nodes);
+  return i;
+}
+
+/** Clear previous cached HSDirs in <b>desc</b>. */
+static void
+service_desc_clear_previous_hsdirs(hs_service_descriptor_t *desc)
+{
+  if (BUG(!desc->previous_hsdirs)) {
+    return;
+  }
+
+  SMARTLIST_FOREACH(desc->previous_hsdirs, char*, s, tor_free(s));
+  smartlist_clear(desc->previous_hsdirs);
+}
+
+/** Note that we attempted to upload <b>desc</b> to <b>hsdir</b>. */
+static void
+service_desc_note_upload(hs_service_descriptor_t *desc, const node_t *hsdir)
+{
+  char b64_digest[BASE64_DIGEST_LEN+1] = {0};
+  digest_to_base64(b64_digest, hsdir->identity);
+
+  if (BUG(!desc->previous_hsdirs)) {
+    return;
+  }
+
+  if (!smartlist_contains_string(desc->previous_hsdirs, b64_digest)) {
+    smartlist_add_strdup(desc->previous_hsdirs, b64_digest);
+  }
+}
+
+/** Schedule an upload of <b>desc</b>. If <b>descriptor_changed</b> is set, it
+ *  means that this descriptor is dirty. */
+STATIC void
+service_desc_schedule_upload(hs_service_descriptor_t *desc,
+                             time_t now,
+                             int descriptor_changed)
+
+{
+  desc->next_upload_time = now;
+
+  /* If the descriptor changed, clean up the old HSDirs list. We want to
+   * re-upload no matter what. */
+  if (descriptor_changed) {
+    service_desc_clear_previous_hsdirs(desc);
+  }
+}
+
+/* Update the given descriptor from the given service. The possible update
+ * actions includes:
+ *    - Picking missing intro points if needed.
+ *    - Incrementing the revision counter if needed.
+ */
+static void
+update_service_descriptor(hs_service_t *service,
+                          hs_service_descriptor_t *desc, time_t now)
+{
+  unsigned int num_intro_points;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(desc->desc);
+
+  num_intro_points = digest256map_size(desc->intro_points.map);
+
+  /* Pick any missing introduction point(s). */
+  if (num_intro_points < service->config.num_intro_points) {
+    unsigned int num_new_intro_points = pick_needed_intro_points(service,
+                                                                 desc);
+    if (num_new_intro_points != 0) {
+      log_info(LD_REND, "Service %s just picked %u intro points and wanted "
+                        "%u for %s descriptor. It currently has %d intro "
+                        "points. Launching ESTABLISH_INTRO circuit shortly.",
+               safe_str_client(service->onion_address),
+               num_new_intro_points,
+               service->config.num_intro_points - num_intro_points,
+               (desc == service->desc_current) ? "current" : "next",
+               num_intro_points);
+      /* We'll build those introduction point into the descriptor once we have
+       * confirmation that the circuits are opened and ready. However,
+       * indicate that this descriptor should be uploaded from now on. */
+      service_desc_schedule_upload(desc, now, 1);
+    }
+    /* Were we able to pick all the intro points we needed? If not, we'll
+     * flag the descriptor that it's missing intro points because it
+     * couldn't pick enough which will trigger a descriptor upload. */
+    if ((num_new_intro_points + num_intro_points) <
+        service->config.num_intro_points) {
+      desc->missing_intro_points = 1;
+    }
+  }
+}
+
+/* Update descriptors for each service if needed. */
+STATIC void
+update_all_descriptors(time_t now)
+{
+  FOR_EACH_SERVICE_BEGIN(service) {
+    /* We'll try to update each descriptor that is if certain conditions apply
+     * in order for the descriptor to be updated. */
+    FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+      update_service_descriptor(service, desc, now);
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Return true iff the given intro point has expired that is it has been used
+ * for too long or we've reached our max seen INTRODUCE2 cell. */
+STATIC int
+intro_point_should_expire(const hs_service_intro_point_t *ip,
+                          time_t now)
+{
+  tor_assert(ip);
+
+  if (ip->introduce2_count >= ip->introduce2_max) {
+    goto expired;
+  }
+
+  if (ip->time_to_expire <= now) {
+    goto expired;
+  }
+
+  /* Not expiring. */
+  return 0;
+ expired:
+  return 1;
+}
+
+/* Go over the given set of intro points for each service and remove any
+ * invalid ones. The conditions for removal are:
+ *
+ *    - The node doesn't exists anymore (not in consensus)
+ *                          OR
+ *    - The intro point maximum circuit retry count has been reached and no
+ *      circuit can be found associated with it.
+ *                          OR
+ *    - The intro point has expired and we should pick a new one.
+ *
+ * If an intro point is removed, the circuit (if any) is immediately close.
+ * If a circuit can't be found, the intro point is kept if it hasn't reached
+ * its maximum circuit retry value and thus should be retried.  */
+static void
+cleanup_intro_points(hs_service_t *service, time_t now)
+{
+  /* List of intro points to close. We can't mark the intro circuits for close
+   * in the modify loop because doing so calls
+   * hs_service_intro_circ_has_closed() which does a digest256map_get() on the
+   * intro points map (that we are iterating over). This can't be done in a
+   * single iteration after a MAP_DEL_CURRENT, the object will still be
+   * returned leading to a use-after-free. So, we close the circuits and free
+   * the intro points after the loop if any. */
+  smartlist_t *ips_to_free = smartlist_new();
+
+  tor_assert(service);
+
+  /* For both descriptors, cleanup the intro points. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    /* Go over the current intro points we have, make sure they are still
+     * valid and remove any of them that aren't. */
+    DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
+                                hs_service_intro_point_t *, ip) {
+      const node_t *node = get_node_from_intro_point(ip);
+      int has_expired = intro_point_should_expire(ip, now);
+
+      /* We cleanup an intro point if it has expired or if we do not know the
+       * node_t anymore (removed from our latest consensus) or if we've
+       * reached the maximum number of retry with a non existing circuit. */
+      if (has_expired || node == NULL ||
+          ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
+        log_info(LD_REND, "Intro point %s%s (retried: %u times). "
+                          "Removing it.",
+                 describe_intro_point(ip),
+                 has_expired ? " has expired" :
+                    (node == NULL) ?  " fell off the consensus" : "",
+                 ip->circuit_retries);
+
+        /* We've retried too many times, remember it as a failed intro point
+         * so we don't pick it up again for INTRO_CIRC_RETRY_PERIOD sec. */
+        if (ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
+          remember_failing_intro_point(ip, desc, approx_time());
+        }
+
+        /* Remove intro point from descriptor map and add it to the list of
+         * ips to free for which we'll also try to close the intro circuit. */
+        MAP_DEL_CURRENT(key);
+        smartlist_add(ips_to_free, ip);
+      }
+    } DIGEST256MAP_FOREACH_END;
+  } FOR_EACH_DESCRIPTOR_END;
+
+  /* Go over the intro points to free and close their circuit if any. */
+  SMARTLIST_FOREACH_BEGIN(ips_to_free, hs_service_intro_point_t *, ip) {
+    /* See if we need to close the intro point circuit as well */
+
+    /* XXX: Legacy code does NOT close circuits like this: it keeps the circuit
+     * open until a new descriptor is uploaded and then closed all expiring
+     * intro point circuit. Here, we close immediately and because we just
+     * discarded the intro point, a new one will be selected, a new descriptor
+     * created and uploaded. There is no difference to an attacker between the
+     * timing of a new consensus and intro point rotation (possibly?). */
+    origin_circuit_t *ocirc = hs_circ_service_get_intro_circ(ip);
+    if (ocirc && !TO_CIRCUIT(ocirc)->marked_for_close) {
+      circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+    }
+
+    /* Cleanup the intro point */
+    service_intro_point_free(ip);
+  } SMARTLIST_FOREACH_END(ip);
+
+  smartlist_free(ips_to_free);
+}
+
+/* Set the next rotation time of the descriptors for the given service for the
+ * time now. */
+static void
+set_rotation_time(hs_service_t *service, time_t now)
+{
+  time_t valid_after;
+  const networkstatus_t *ns = networkstatus_get_live_consensus(now);
+  if (ns) {
+    valid_after = ns->valid_after;
+  } else {
+    valid_after = now;
+  }
+
+  tor_assert(service);
+  service->state.next_rotation_time =
+    sr_state_get_start_time_of_current_protocol_run(valid_after) +
+    sr_state_get_protocol_run_duration();
+
+  {
+    char fmt_time[ISO_TIME_LEN + 1];
+    format_local_iso_time(fmt_time, service->state.next_rotation_time);
+    log_info(LD_REND, "Next descriptor rotation time set to %s for %s",
+             fmt_time, safe_str_client(service->onion_address));
+  }
+}
+
+/* Return true iff the service should rotate its descriptor. The time now is
+ * only used to fetch the live consensus and if none can be found, this
+ * returns false. */
+static unsigned int
+should_rotate_descriptors(hs_service_t *service, time_t now)
+{
+  const networkstatus_t *ns;
+
+  tor_assert(service);
+
+  ns = networkstatus_get_live_consensus(now);
+  if (ns == NULL) {
+    goto no_rotation;
+  }
+
+  if (ns->valid_after >= service->state.next_rotation_time) {
+    /* In theory, we should never get here with no descriptors. We can never
+     * have a NULL current descriptor except when tor starts up. The next
+     * descriptor can be NULL after a rotation but we build a new one right
+     * after.
+     *
+     * So, when tor starts, the next rotation time is set to the start of the
+     * next SRV period using the consensus valid after time so it should
+     * always be set to a future time value. This means that we should never
+     * reach this point at bootup that is this check safeguards tor in never
+     * allowing a rotation if the valid after time is smaller than the next
+     * rotation time.
+     *
+     * This is all good in theory but we've had a NULL descriptor issue here
+     * so this is why we BUG() on both with extra logging to try to understand
+     * how this can possibly happens. We'll simply ignore and tor should
+     * recover from this by skipping rotation and building the missing
+     * descriptors just after this. */
+    if (BUG(service->desc_current == NULL || service->desc_next == NULL)) {
+      log_warn(LD_BUG, "Service descriptor is NULL (%p/%p). Next rotation "
+                       "time is %ld (now: %ld). Valid after time from "
+                       "consensus is %ld",
+               service->desc_current, service->desc_next,
+               (long)service->state.next_rotation_time,
+               (long)now,
+               (long)ns->valid_after);
+      goto no_rotation;
+    }
+    goto rotation;
+  }
+
+ no_rotation:
+  return 0;
+ rotation:
+  return 1;
+}
+
+/* Rotate the service descriptors of the given service. The current descriptor
+ * will be freed, the next one put in as the current and finally the next
+ * descriptor pointer is NULLified. */
+static void
+rotate_service_descriptors(hs_service_t *service, time_t now)
+{
+  if (service->desc_current) {
+    /* Close all IP circuits for the descriptor. */
+    close_intro_circuits(&service->desc_current->intro_points);
+    /* We don't need this one anymore, we won't serve any clients coming with
+     * this service descriptor. */
+    service_descriptor_free(service->desc_current);
+  }
+  /* The next one become the current one and emptying the next will trigger
+   * a descriptor creation for it. */
+  service->desc_current = service->desc_next;
+  service->desc_next = NULL;
+
+  /* We've just rotated, set the next time for the rotation. */
+  set_rotation_time(service, now);
+}
+
+/* Rotate descriptors for each service if needed. A non existing current
+ * descriptor will trigger a descriptor build for the next time period. */
+STATIC void
+rotate_all_descriptors(time_t now)
+{
+  /* XXX We rotate all our service descriptors at once. In the future it might
+   *     be wise, to rotate service descriptors independently to hide that all
+   *     those descriptors are on the same tor instance */
+
+  FOR_EACH_SERVICE_BEGIN(service) {
+
+    /* Note for a service booting up: Both descriptors are NULL in that case
+     * so this function might return true if we are in the timeframe for a
+     * rotation leading to basically swapping two NULL pointers which is
+     * harmless. However, the side effect is that triggering a rotation will
+     * update the service state and avoid doing anymore rotations after the
+     * two descriptors have been built. */
+    if (!should_rotate_descriptors(service, now)) {
+      continue;
+    }
+
+    log_info(LD_REND, "Time to rotate our descriptors (%p / %p) for %s",
+             service->desc_current, service->desc_next,
+             safe_str_client(service->onion_address));
+
+    rotate_service_descriptors(service, now);
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Scheduled event run from the main loop. Make sure all our services are up
+ * to date and ready for the other scheduled events. This includes looking at
+ * the introduction points status and descriptor rotation time. */
+STATIC void
+run_housekeeping_event(time_t now)
+{
+  /* Note that nothing here opens circuit(s) nor uploads descriptor(s). We are
+   * simply moving things around or removing unneeded elements. */
+
+  FOR_EACH_SERVICE_BEGIN(service) {
+
+    /* If the service is starting off, set the rotation time. We can't do that
+     * at configure time because the get_options() needs to be set for setting
+     * that time that uses the voting interval. */
+    if (service->state.next_rotation_time == 0) {
+      /* Set the next rotation time of the descriptors. If it's Oct 25th
+       * 23:47:00, the next rotation time is when the next SRV is computed
+       * which is at Oct 26th 00:00:00 that is in 13 minutes. */
+      set_rotation_time(service, now);
+    }
+
+    /* Cleanup invalid intro points from the service descriptor. */
+    cleanup_intro_points(service, now);
+
+    /* Remove expired failing intro point from the descriptor failed list. We
+     * reset them at each INTRO_CIRC_RETRY_PERIOD. */
+    remove_expired_failing_intro(service, now);
+
+    /* At this point, the service is now ready to go through the scheduled
+     * events guaranteeing a valid state. Intro points might be missing from
+     * the descriptors after the cleanup but the update/build process will
+     * make sure we pick those missing ones. */
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Scheduled event run from the main loop. Make sure all descriptors are up to
+ * date. Once this returns, each service descriptor needs to be considered for
+ * new introduction circuits and then for upload. */
+static void
+run_build_descriptor_event(time_t now)
+{
+  /* For v2 services, this step happens in the upload event. */
+
+  /* Run v3+ events. */
+  /* We start by rotating the descriptors only if needed. */
+  rotate_all_descriptors(now);
+
+  /* Then, we'll try to build  new descriptors that we might need. The
+   * condition is that the next descriptor is non existing because it has
+   * been rotated or we just started up. */
+  build_all_descriptors(now);
+
+  /* Finally, we'll check if we should update the descriptors. Missing
+   * introduction points will be picked in this function which is useful for
+   * newly built descriptors. */
+  update_all_descriptors(now);
+}
+
+/* For the given service, launch any intro point circuits that could be
+ * needed. This considers every descriptor of the service. */
+static void
+launch_intro_point_circuits(hs_service_t *service)
+{
+  tor_assert(service);
+
+  /* For both descriptors, try to launch any missing introduction point
+   * circuits using the current map. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    /* Keep a ref on if we need a direct connection. We use this often. */
+    unsigned int direct_conn = service->config.is_single_onion;
+
+    DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
+                                hs_service_intro_point_t *, ip) {
+      extend_info_t *ei;
+
+      /* Skip the intro point that already has an existing circuit
+       * (established or not). */
+      if (hs_circ_service_get_intro_circ(ip)) {
+        continue;
+      }
+
+      ei = get_extend_info_from_intro_point(ip, direct_conn);
+      if (ei == NULL) {
+        /* This is possible if we can get a node_t but not the extend info out
+         * of it. In this case, we remove the intro point and a new one will
+         * be picked at the next main loop callback. */
+        MAP_DEL_CURRENT(key);
+        service_intro_point_free(ip);
+        continue;
+      }
+
+      /* Launch a circuit to the intro point. */
+      ip->circuit_retries++;
+      if (hs_circ_launch_intro_point(service, ip, ei) < 0) {
+        log_info(LD_REND, "Unable to launch intro circuit to node %s "
+                          "for service %s.",
+                 safe_str_client(extend_info_describe(ei)),
+                 safe_str_client(service->onion_address));
+        /* Intro point will be retried if possible after this. */
+      }
+      extend_info_free(ei);
+    } DIGEST256MAP_FOREACH_END;
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Don't try to build more than this many circuits before giving up for a
+ * while. Dynamically calculated based on the configured number of intro
+ * points for the given service and how many descriptor exists. The default
+ * use case of 3 introduction points and two descriptors will allow 28
+ * circuits for a retry period (((3 + 2) + (3 * 3)) * 2). */
+static unsigned int
+get_max_intro_circ_per_period(const hs_service_t *service)
+{
+  unsigned int count = 0;
+  unsigned int multiplier = 0;
+  unsigned int num_wanted_ip;
+
+  tor_assert(service);
+  tor_assert(service->config.num_intro_points <=
+             HS_CONFIG_V3_MAX_INTRO_POINTS);
+
+/* For a testing network, allow to do it for the maximum amount so circuit
+ * creation and rotation and so on can actually be tested without limit. */
+#define MAX_INTRO_POINT_CIRCUIT_RETRIES_TESTING -1
+  if (get_options()->TestingTorNetwork) {
+    return MAX_INTRO_POINT_CIRCUIT_RETRIES_TESTING;
+  }
+
+  num_wanted_ip = service->config.num_intro_points;
+
+  /* The calculation is as follow. We have a number of intro points that we
+   * want configured as a torrc option (num_intro_points). We then add an
+   * extra value so we can launch multiple circuits at once and pick the
+   * quickest ones. For instance, we want 3 intros, we add 2 extra so we'll
+   * pick 5 intros and launch 5 circuits. */
+  count += (num_wanted_ip + get_intro_point_num_extra());
+
+  /* Then we add the number of retries that is possible to do for each intro
+   * point. If we want 3 intros, we'll allow 3 times the number of possible
+   * retry. */
+  count += (num_wanted_ip * MAX_INTRO_POINT_CIRCUIT_RETRIES);
+
+  /* Then, we multiply by a factor of 2 if we have both descriptor or 0 if we
+   * have none.  */
+  multiplier += (service->desc_current) ? 1 : 0;
+  multiplier += (service->desc_next) ? 1 : 0;
+
+  return (count * multiplier);
+}
+
+/* For the given service, return 1 if the service is allowed to launch more
+ * introduction circuits else 0 if the maximum has been reached for the retry
+ * period of INTRO_CIRC_RETRY_PERIOD. */
+STATIC int
+can_service_launch_intro_circuit(hs_service_t *service, time_t now)
+{
+  tor_assert(service);
+
+  /* Consider the intro circuit retry period of the service. */
+  if (now > (service->state.intro_circ_retry_started_time +
+             INTRO_CIRC_RETRY_PERIOD)) {
+    service->state.intro_circ_retry_started_time = now;
+    service->state.num_intro_circ_launched = 0;
+    goto allow;
+  }
+  /* Check if we can still launch more circuits in this period. */
+  if (service->state.num_intro_circ_launched <=
+      get_max_intro_circ_per_period(service)) {
+    goto allow;
+  }
+
+  /* Rate limit log that we've reached our circuit creation limit. */
+  {
+    char *msg;
+    time_t elapsed_time = now - service->state.intro_circ_retry_started_time;
+    static ratelim_t rlimit = RATELIM_INIT(INTRO_CIRC_RETRY_PERIOD);
+    if ((msg = rate_limit_log(&rlimit, now))) {
+      log_info(LD_REND, "Hidden service %s exceeded its circuit launch limit "
+                        "of %u per %d seconds. It launched %u circuits in "
+                        "the last %ld seconds. Will retry in %ld seconds.",
+               safe_str_client(service->onion_address),
+               get_max_intro_circ_per_period(service),
+               INTRO_CIRC_RETRY_PERIOD,
+               service->state.num_intro_circ_launched,
+               (long int) elapsed_time,
+               (long int) (INTRO_CIRC_RETRY_PERIOD - elapsed_time));
+      tor_free(msg);
+    }
+  }
+
+  /* Not allow. */
+  return 0;
+ allow:
+  return 1;
+}
+
+/* Scheduled event run from the main loop. Make sure we have all the circuits
+ * we need for each service. */
+static void
+run_build_circuit_event(time_t now)
+{
+  /* Make sure we can actually have enough information or able to build
+   * internal circuits as required by services. */
+  if (router_have_consensus_path() == CONSENSUS_PATH_UNKNOWN ||
+      !have_completed_a_circuit()) {
+    return;
+  }
+
+  /* Run v2 check. */
+  if (rend_num_services() > 0) {
+    rend_consider_services_intro_points(now);
+  }
+
+  /* Run v3+ check. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    /* For introduction circuit, we need to make sure we don't stress too much
+     * circuit creation so make sure this service is respecting that limit. */
+    if (can_service_launch_intro_circuit(service, now)) {
+      /* Launch intro point circuits if needed. */
+      launch_intro_point_circuits(service);
+      /* Once the circuits have opened, we'll make sure to update the
+       * descriptor intro point list and cleanup any extraneous. */
+    }
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Encode and sign the service descriptor desc and upload it to the given
+ * hidden service directory.  This does nothing if PublishHidServDescriptors
+ * is false. */
+static void
+upload_descriptor_to_hsdir(const hs_service_t *service,
+                           hs_service_descriptor_t *desc, const node_t *hsdir)
+{
+  char *encoded_desc = NULL;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(hsdir);
+
+  /* Let's avoid doing that if tor is configured to not publish. */
+  if (!get_options()->PublishHidServDescriptors) {
+    log_info(LD_REND, "Service %s not publishing descriptor. "
+                      "PublishHidServDescriptors is set to 1.",
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+
+  /* First of all, we'll encode the descriptor. This should NEVER fail but
+   * just in case, let's make sure we have an actual usable descriptor. */
+  if (BUG(hs_desc_encode_descriptor(desc->desc, &desc->signing_kp,
+                                    &encoded_desc) < 0)) {
+    goto end;
+  }
+
+  /* Time to upload the descriptor to the directory. */
+  hs_service_upload_desc_to_dir(encoded_desc, service->config.version,
+                                &service->keys.identity_pk,
+                                &desc->blinded_kp.pubkey, hsdir->rs);
+
+  /* Add this node to previous_hsdirs list */
+  service_desc_note_upload(desc, hsdir);
+
+  /* Logging so we know where it was sent. */
+  {
+    int is_next_desc = (service->desc_next == desc);
+    const uint8_t *idx = (is_next_desc) ? hsdir->hsdir_index.store_second:
+                                          hsdir->hsdir_index.store_first;
+    log_info(LD_REND, "Service %s %s descriptor of revision %" PRIu64
+                      " initiated upload request to %s with index %s",
+             safe_str_client(service->onion_address),
+             (is_next_desc) ? "next" : "current",
+             desc->desc->plaintext_data.revision_counter,
+             safe_str_client(node_describe(hsdir)),
+             safe_str_client(hex_str((const char *) idx, 32)));
+
+    /* Fire a UPLOAD control port event. */
+    hs_control_desc_event_upload(service->onion_address, hsdir->identity,
+                                 &desc->blinded_kp.pubkey, idx);
+  }
+
+ end:
+  tor_free(encoded_desc);
+  return;
+}
+
+/** Return a newly-allocated string for our state file which contains revision
+ *  counter information for <b>desc</b>. The format is:
+ *
+ *     HidServRevCounter <blinded_pubkey> <rev_counter>
+ */
+STATIC char *
+encode_desc_rev_counter_for_state(const hs_service_descriptor_t *desc)
+{
+  char *state_str = NULL;
+  char blinded_pubkey_b64[ED25519_BASE64_LEN+1];
+  uint64_t rev_counter = desc->desc->plaintext_data.revision_counter;
+  const ed25519_public_key_t *blinded_pubkey = &desc->blinded_kp.pubkey;
+
+  /* Turn the blinded key into b64 so that we save it on state */
+  tor_assert(blinded_pubkey);
+  if (ed25519_public_to_base64(blinded_pubkey_b64, blinded_pubkey) < 0) {
+    goto done;
+  }
+
+  /* Format is: <blinded key> <rev counter> */
+  tor_asprintf(&state_str, "%s %" PRIu64, blinded_pubkey_b64, rev_counter);
+
+  log_info(LD_GENERAL, "[!] Adding rev counter %" PRIu64 " for %s!",
+           rev_counter, blinded_pubkey_b64);
+
+ done:
+  return state_str;
+}
+
+/** Update HS descriptor revision counters in our state by removing the old
+ *  ones and writing down the ones that are currently active. */
+static void
+update_revision_counters_in_state(void)
+{
+  config_line_t *lines = NULL;
+  config_line_t **nextline = &lines;
+  or_state_t *state = get_or_state();
+
+  /* Prepare our state structure with the rev counters */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+      /* We don't want to save zero counters */
+      if (desc->desc->plaintext_data.revision_counter == 0) {
+        continue;
+      }
+
+      *nextline = tor_malloc_zero(sizeof(config_line_t));
+      (*nextline)->key = tor_strdup("HidServRevCounter");
+      (*nextline)->value = encode_desc_rev_counter_for_state(desc);
+      nextline = &(*nextline)->next;
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+
+  /* Remove the old rev counters, and replace them with the new ones */
+  config_free_lines(state->HidServRevCounter);
+  state->HidServRevCounter = lines;
+
+  /* Set the state as dirty since we just edited it */
+  if (!get_options()->AvoidDiskWrites) {
+    or_state_mark_dirty(state, 0);
+  }
+}
+
+/** Scan the string <b>state_line</b> for the revision counter of the service
+ *  with <b>blinded_pubkey</b>. Set <b>service_found_out</b> to True if the
+ *  line is relevant to this service, and return the cached revision
+ *  counter. Else set <b>service_found_out</b> to False. */
+STATIC uint64_t
+check_state_line_for_service_rev_counter(const char *state_line,
+                                    const ed25519_public_key_t *blinded_pubkey,
+                                    int *service_found_out)
+{
+  smartlist_t *items = NULL;
+  int ok;
+  ed25519_public_key_t pubkey_in_state;
+  uint64_t rev_counter = 0;
+
+  tor_assert(service_found_out);
+  tor_assert(state_line);
+  tor_assert(blinded_pubkey);
+
+  /* Assume that the line is not for this service */
+  *service_found_out = 0;
+
+  /* Start parsing the state line */
+  items = smartlist_new();
+  smartlist_split_string(items, state_line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  if (smartlist_len(items) < 2) {
+    log_warn(LD_GENERAL, "Incomplete rev counter line. Ignoring.");
+    goto done;
+  }
+
+  char *b64_key_str = smartlist_get(items, 0);
+  char *saved_rev_counter_str = smartlist_get(items, 1);
+
+  /* Parse blinded key to check if it's for this hidden service */
+  if (ed25519_public_from_base64(&pubkey_in_state, b64_key_str) < 0) {
+    log_warn(LD_GENERAL, "Unable to base64 key in revcount line. Ignoring.");
+    goto done;
+  }
+  /* State line not for this hidden service */
+  if (!ed25519_pubkey_eq(&pubkey_in_state, blinded_pubkey)) {
+    goto done;
+  }
+
+  rev_counter = tor_parse_uint64(saved_rev_counter_str,
+                                 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_GENERAL, "Unable to parse rev counter. Ignoring.");
+    goto done;
+  }
+
+  /* Since we got this far, the line was for this service */
+  *service_found_out = 1;
+
+  log_info(LD_GENERAL, "Found rev counter for %s: %" PRIu64,
+           b64_key_str, rev_counter);
+
+ done:
+  tor_assert(items);
+  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
+  smartlist_free(items);
+
+  return rev_counter;
+}
+
+/** Dig into our state file and find the current revision counter for the
+ *  service with blinded key <b>blinded_pubkey</b>. If no revision counter is
+ *  found, return 0. */
+static uint64_t
+get_rev_counter_for_service(const ed25519_public_key_t *blinded_pubkey)
+{
+  or_state_t *state = get_or_state();
+  config_line_t *line;
+
+  /* Set default value for rev counters (if not found) to 0 */
+  uint64_t final_rev_counter = 0;
+
+  for (line = state->HidServRevCounter ; line ; line = line->next) {
+    int service_found = 0;
+    uint64_t rev_counter = 0;
+
+    tor_assert(!strcmp(line->key, "HidServRevCounter"));
+
+    /* Scan all the HidServRevCounter lines till we find the line for this
+       service: */
+    rev_counter = check_state_line_for_service_rev_counter(line->value,
+                                                           blinded_pubkey,
+                                                           &service_found);
+    if (service_found) {
+      final_rev_counter = rev_counter;
+      goto done;
+    }
+  }
+
+ done:
+  return final_rev_counter;
+}
+
+/** Update the value of the revision counter for <b>hs_desc</b> and save it on
+    our state file. */
+static void
+increment_descriptor_revision_counter(hs_descriptor_t *hs_desc)
+{
+  /* Find stored rev counter if it exists */
+  uint64_t rev_counter =
+    get_rev_counter_for_service(&hs_desc->plaintext_data.blinded_pubkey);
+
+  /* Increment the revision counter of <b>hs_desc</b> so the next update (which
+   * will trigger an upload) will have the right value. We do this at this
+   * stage to only do it once because a descriptor can have many updates before
+   * being uploaded. By doing it at upload, we are sure to only increment by 1
+   * and thus avoid leaking how many operations we made on the descriptor from
+   * the previous one before uploading. */
+  rev_counter++;
+  hs_desc->plaintext_data.revision_counter = rev_counter;
+
+  update_revision_counters_in_state();
+}
+
+/** Set the revision counter in <b>hs_desc</b>, using the state file to find
+ *  the current counter value if it exists. */
+static void
+set_descriptor_revision_counter(hs_descriptor_t *hs_desc)
+{
+  /* Find stored rev counter if it exists */
+  uint64_t rev_counter =
+    get_rev_counter_for_service(&hs_desc->plaintext_data.blinded_pubkey);
+
+  hs_desc->plaintext_data.revision_counter = rev_counter;
+}
+
+/* Encode and sign the service descriptor desc and upload it to the
+ * responsible hidden service directories. If for_next_period is true, the set
+ * of directories are selected using the next hsdir_index. This does nothing
+ * if PublishHidServDescriptors is false. */
+STATIC void
+upload_descriptor_to_all(const hs_service_t *service,
+                         hs_service_descriptor_t *desc)
+{
+  smartlist_t *responsible_dirs = NULL;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* We'll first cancel any directory request that are ongoing for this
+   * descriptor. It is possible that we can trigger multiple uploads in a
+   * short time frame which can lead to a race where the second upload arrives
+   * before the first one leading to a 400 malformed descriptor response from
+   * the directory. Closing all pending requests avoids that. */
+  close_directory_connections(service, desc);
+
+  /* Get our list of responsible HSDir. */
+  responsible_dirs = smartlist_new();
+  /* The parameter 0 means that we aren't a client so tell the function to use
+   * the spread store consensus paremeter. */
+  hs_get_responsible_hsdirs(&desc->blinded_kp.pubkey, desc->time_period_num,
+                            service->desc_next == desc, 0, responsible_dirs);
+
+  /** Clear list of previous hsdirs since we are about to upload to a new
+   *  list. Let's keep it up to date. */
+  service_desc_clear_previous_hsdirs(desc);
+
+  /* For each responsible HSDir we have, initiate an upload command. */
+  SMARTLIST_FOREACH_BEGIN(responsible_dirs, const routerstatus_t *,
+                          hsdir_rs) {
+    const node_t *hsdir_node = node_get_by_id(hsdir_rs->identity_digest);
+    /* Getting responsible hsdir implies that the node_t object exists for the
+     * routerstatus_t found in the consensus else we have a problem. */
+    tor_assert(hsdir_node);
+    /* Upload this descriptor to the chosen directory. */
+    upload_descriptor_to_hsdir(service, desc, hsdir_node);
+  } SMARTLIST_FOREACH_END(hsdir_rs);
+
+  /* Set the next upload time for this descriptor. Even if we are configured
+   * to not upload, we still want to follow the right cycle of life for this
+   * descriptor. */
+  desc->next_upload_time =
+    (time(NULL) + crypto_rand_int_range(HS_SERVICE_NEXT_UPLOAD_TIME_MIN,
+                                        HS_SERVICE_NEXT_UPLOAD_TIME_MAX));
+  {
+    char fmt_next_time[ISO_TIME_LEN+1];
+    format_local_iso_time(fmt_next_time, desc->next_upload_time);
+    log_debug(LD_REND, "Service %s set to upload a descriptor at %s",
+              safe_str_client(service->onion_address), fmt_next_time);
+  }
+
+  /* Update the revision counter of this descriptor */
+  increment_descriptor_revision_counter(desc->desc);
+
+  smartlist_free(responsible_dirs);
+  return;
+}
+
+/** The set of HSDirs have changed: check if the change affects our descriptor
+ *  HSDir placement, and if it does, reupload the desc. */
+STATIC int
+service_desc_hsdirs_changed(const hs_service_t *service,
+                            const hs_service_descriptor_t *desc)
+{
+  int should_reupload = 0;
+  smartlist_t *responsible_dirs = smartlist_new();
+
+  /* No desc upload has happened yet: it will happen eventually */
+  if (!desc->previous_hsdirs || !smartlist_len(desc->previous_hsdirs)) {
+    goto done;
+  }
+
+  /* Get list of responsible hsdirs */
+  hs_get_responsible_hsdirs(&desc->blinded_kp.pubkey, desc->time_period_num,
+                            service->desc_next == desc, 0, responsible_dirs);
+
+  /* Check if any new hsdirs have been added to the responsible hsdirs set:
+   * Iterate over the list of new hsdirs, and reupload if any of them is not
+   * present in the list of previous hsdirs.
+   */
+  SMARTLIST_FOREACH_BEGIN(responsible_dirs, const routerstatus_t *, hsdir_rs) {
+    char b64_digest[BASE64_DIGEST_LEN+1] = {0};
+    digest_to_base64(b64_digest, hsdir_rs->identity_digest);
+
+    if (!smartlist_contains_string(desc->previous_hsdirs, b64_digest)) {
+      should_reupload = 1;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(hsdir_rs);
+
+ done:
+  smartlist_free(responsible_dirs);
+
+  return should_reupload;
+}
+
+/* Return 1 if the given descriptor from the given service can be uploaded
+ * else return 0 if it can not. */
+static int
+should_service_upload_descriptor(const hs_service_t *service,
+                              const hs_service_descriptor_t *desc, time_t now)
+{
+  unsigned int num_intro_points;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* If this descriptors has missing intro points that is that it couldn't get
+   * them all when it was time to pick them, it means that we should upload
+   * instead of waiting an arbitrary amount of time breaking the service.
+   * Else, if we have no missing intro points, we use the value taken from the
+   * service configuration. */
+  if (desc->missing_intro_points) {
+    num_intro_points = digest256map_size(desc->intro_points.map);
+  } else {
+    num_intro_points = service->config.num_intro_points;
+  }
+
+  /* This means we tried to pick intro points but couldn't get any so do not
+   * upload descriptor in this case. We need at least one for the service to
+   * be reachable. */
+  if (desc->missing_intro_points && num_intro_points == 0) {
+    goto cannot;
+  }
+
+  /* Check if all our introduction circuit have been established for all the
+   * intro points we have selected. */
+  if (count_desc_circuit_established(desc) != num_intro_points) {
+    goto cannot;
+  }
+
+  /* Is it the right time to upload? */
+  if (desc->next_upload_time > now) {
+    goto cannot;
+  }
+
+  /* Don't upload desc if we don't have a live consensus */
+  if (!networkstatus_get_live_consensus(now)) {
+    goto cannot;
+  }
+
+  /* Do we know enough router descriptors to have adequate vision of the HSDir
+     hash ring? */
+  if (!router_have_minimum_dir_info()) {
+    goto cannot;
+  }
+
+  /* Can upload! */
+  return 1;
+ cannot:
+  return 0;
+}
+
+/* Scheduled event run from the main loop. Try to upload the descriptor for
+ * each service. */
+STATIC void
+run_upload_descriptor_event(time_t now)
+{
+  /* v2 services use the same function for descriptor creation and upload so
+   * we do everything here because the intro circuits were checked before. */
+  if (rend_num_services() > 0) {
+    rend_consider_services_upload(now);
+    rend_consider_descriptor_republication();
+  }
+
+  /* Run v3+ check. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+      /* If we were asked to re-examine the hash ring, and it changed, then
+         schedule an upload */
+      if (consider_republishing_hs_descriptors &&
+          service_desc_hsdirs_changed(service, desc)) {
+        service_desc_schedule_upload(desc, now, 0);
+      }
+
+      /* Can this descriptor be uploaded? */
+      if (!should_service_upload_descriptor(service, desc, now)) {
+        continue;
+      }
+
+      log_info(LD_REND, "Initiating upload for hidden service %s descriptor "
+                        "for service %s with %u/%u introduction points%s.",
+               (desc == service->desc_current) ? "current" : "next",
+               safe_str_client(service->onion_address),
+               digest256map_size(desc->intro_points.map),
+               service->config.num_intro_points,
+               (desc->missing_intro_points) ? " (couldn't pick more)" : "");
+
+      /* At this point, we have to upload the descriptor so start by building
+       * the intro points descriptor section which we are now sure to be
+       * accurate because all circuits have been established. */
+      build_desc_intro_points(service, desc, now);
+
+      upload_descriptor_to_all(service, desc);
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+
+  /* We are done considering whether to republish rend descriptors */
+  consider_republishing_hs_descriptors = 0;
+}
+
+/* Called when the introduction point circuit is done building and ready to be
+ * used. */
+static void
+service_intro_circ_has_opened(origin_circuit_t *circ)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_descriptor_t *desc = NULL;
+
+  tor_assert(circ);
+
+  /* Let's do some basic sanity checking of the circ state */
+  if (BUG(!circ->cpath)) {
+    return;
+  }
+  if (BUG(TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)) {
+    return;
+  }
+  if (BUG(!circ->hs_ident)) {
+    return;
+  }
+
+  /* Get the corresponding service and intro point. */
+  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
+
+  if (service == NULL) {
+    log_warn(LD_REND, "Unknown service identity key %s on the introduction "
+                      "circuit %u. Can't find onion service.",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+  if (ip == NULL) {
+    log_warn(LD_REND, "Unknown introduction point auth key on circuit %u "
+                      "for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+  /* We can't have an IP object without a descriptor. */
+  tor_assert(desc);
+
+  if (hs_circ_service_intro_has_opened(service, ip, desc, circ)) {
+    /* Getting here means that the circuit has been re-purposed because we
+     * have enough intro circuit opened. Remove the IP from the service. */
+    service_intro_point_remove(service, ip);
+    service_intro_point_free(ip);
+  }
+
+  goto done;
+
+ err:
+  /* Close circuit, we can't use it. */
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NOSUCHSERVICE);
+ done:
+  return;
+}
+
+/* Called when a rendezvous circuit is done building and ready to be used. */
+static void
+service_rendezvous_circ_has_opened(origin_circuit_t *circ)
+{
+  hs_service_t *service = NULL;
+
+  tor_assert(circ);
+  tor_assert(circ->cpath);
+  /* Getting here means this is a v3 rendezvous circuit. */
+  tor_assert(circ->hs_ident);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* Declare the circuit dirty to avoid reuse, and for path-bias. We set the
+   * timestamp regardless of its content because that circuit could have been
+   * cannibalized so in any cases, we are about to use that circuit more. */
+  TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+  pathbias_count_use_attempt(circ);
+
+  /* Get the corresponding service and intro point. */
+  get_objects_from_ident(circ->hs_ident, &service, NULL, NULL);
+  if (service == NULL) {
+    log_warn(LD_REND, "Unknown service identity key %s on the rendezvous "
+                      "circuit %u with cookie %s. Can't find onion service.",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id,
+             hex_str((const char *) circ->hs_ident->rendezvous_cookie,
+                     REND_COOKIE_LEN));
+    goto err;
+  }
+
+  /* If the cell can't be sent, the circuit will be closed within this
+   * function. */
+  hs_circ_service_rp_has_opened(service, circ);
+  goto done;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NOSUCHSERVICE);
+ done:
+  return;
+}
+
+/* We've been expecting an INTRO_ESTABLISHED cell on this circuit and it just
+ * arrived. Handle the INTRO_ESTABLISHED cell arriving on the given
+ * introduction circuit. Return 0 on success else a negative value. */
+static int
+service_handle_intro_established(origin_circuit_t *circ,
+                                 const uint8_t *payload,
+                                 size_t payload_len)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+
+  tor_assert(circ);
+  tor_assert(payload);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  /* We need the service and intro point for this cell. */
+  get_objects_from_ident(circ->hs_ident, &service, &ip, NULL);
+
+  /* Get service object from the circuit identifier. */
+  if (service == NULL) {
+    log_warn(LD_REND, "Unknown service identity key %s on the introduction "
+                      "circuit %u. Can't find onion service.",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+  if (ip == NULL) {
+    /* We don't recognize the key. */
+    log_warn(LD_REND, "Introduction circuit established without an intro "
+                      "point object on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  /* Try to parse the payload into a cell making sure we do actually have a
+   * valid cell. On success, the ip object and circuit purpose is updated to
+   * reflect the fact that the introduction circuit is established. */
+  if (hs_circ_handle_intro_established(service, ip, circ, payload,
+                                       payload_len) < 0) {
+    goto err;
+  }
+
+  /* Flag that we have an established circuit for this intro point. This value
+   * is what indicates the upload scheduled event if we are ready to build the
+   * intro point into the descriptor and upload. */
+  ip->circuit_established = 1;
+
+  log_info(LD_REND, "Successfully received an INTRO_ESTABLISHED cell "
+                    "on circuit %u for service %s",
+           TO_CIRCUIT(circ)->n_circ_id,
+           safe_str_client(service->onion_address));
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* We just received an INTRODUCE2 cell on the established introduction circuit
+ * circ. Handle the cell and return 0 on success else a negative value. */
+static int
+service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload,
+                          size_t payload_len)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_descriptor_t *desc = NULL;
+
+  tor_assert(circ);
+  tor_assert(payload);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO);
+
+  /* We'll need every object associated with this circuit. */
+  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
+
+  /* Get service object from the circuit identifier. */
+  if (service == NULL) {
+    log_warn(LD_BUG, "Unknown service identity key %s when handling "
+                     "an INTRODUCE2 cell on circuit %u",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+  if (ip == NULL) {
+    /* We don't recognize the key. */
+    log_warn(LD_BUG, "Unknown introduction auth key when handling "
+                     "an INTRODUCE2 cell on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+  /* If we have an IP object, we MUST have a descriptor object. */
+  tor_assert(desc);
+
+  /* The following will parse, decode and launch the rendezvous point circuit.
+   * Both current and legacy cells are handled. */
+  if (hs_circ_handle_introduce2(service, circ, ip, desc->desc->subcredential,
+                                payload, payload_len) < 0) {
+    goto err;
+  }
+
+  return 0;
+ err:
+  return -1;
+}
+
+/* Add to list every filename used by service. This is used by the sandbox
+ * subsystem. */
+static void
+service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list)
+{
+  const char *s_dir;
+  char fname[128] = {0};
+
+  tor_assert(service);
+  tor_assert(list);
+
+  /* Ease our life. */
+  s_dir = service->config.directory_path;
+  /* The hostname file. */
+  smartlist_add(list, hs_path_from_filename(s_dir, fname_hostname));
+  /* The key files splitted in two. */
+  tor_snprintf(fname, sizeof(fname), "%s_secret_key", fname_keyfile_prefix);
+  smartlist_add(list, hs_path_from_filename(s_dir, fname));
+  tor_snprintf(fname, sizeof(fname), "%s_public_key", fname_keyfile_prefix);
+  smartlist_add(list, hs_path_from_filename(s_dir, fname));
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/* This is called everytime the service map (v2 or v3) changes that is if an
+ * element is added or removed. */
+void
+hs_service_map_has_changed(void)
+{
+  /* If we now have services where previously we had not, we need to enable
+   * the HS service main loop event. If we changed to having no services, we
+   * need to disable the event. */
+  rescan_periodic_events(get_options());
+}
+
+/* Upload an encoded descriptor in encoded_desc of the given version. This
+ * descriptor is for the service identity_pk and blinded_pk used to setup the
+ * directory connection identifier. It is uploaded to the directory hsdir_rs
+ * routerstatus_t object.
+ *
+ * NOTE: This function does NOT check for PublishHidServDescriptors because it
+ * is only used by the control port command HSPOST outside of this subsystem.
+ * Inside this code, upload_descriptor_to_hsdir() should be used. */
+void
+hs_service_upload_desc_to_dir(const char *encoded_desc,
+                              const uint8_t version,
+                              const ed25519_public_key_t *identity_pk,
+                              const ed25519_public_key_t *blinded_pk,
+                              const routerstatus_t *hsdir_rs)
+{
+  char version_str[4] = {0};
+  directory_request_t *dir_req;
+  hs_ident_dir_conn_t ident;
+
+  tor_assert(encoded_desc);
+  tor_assert(identity_pk);
+  tor_assert(blinded_pk);
+  tor_assert(hsdir_rs);
+
+  /* Setup the connection identifier. */
+  memset(&ident, 0, sizeof(ident));
+  hs_ident_dir_conn_init(identity_pk, blinded_pk, &ident);
+
+  /* This is our resource when uploading which is used to construct the URL
+   * with the version number: "/tor/hs/<version>/publish". */
+  tor_snprintf(version_str, sizeof(version_str), "%u", version);
+
+  /* Build the directory request for this HSDir. */
+  dir_req = directory_request_new(DIR_PURPOSE_UPLOAD_HSDESC);
+  directory_request_set_routerstatus(dir_req, hsdir_rs);
+  directory_request_set_indirection(dir_req, DIRIND_ANONYMOUS);
+  directory_request_set_resource(dir_req, version_str);
+  directory_request_set_payload(dir_req, encoded_desc,
+                                strlen(encoded_desc));
+  /* The ident object is copied over the directory connection object once
+   * the directory request is initiated. */
+  directory_request_upload_set_hs_ident(dir_req, &ident);
+
+  /* Initiate the directory request to the hsdir.*/
+  directory_initiate_request(dir_req);
+  directory_request_free(dir_req);
+}
+
+/* Add the ephemeral service using the secret key sk and ports. Both max
+ * streams parameter will be set in the newly created service.
+ *
+ * Ownership of sk and ports is passed to this routine.  Regardless of
+ * success/failure, callers should not touch these values after calling this
+ * routine, and may assume that correct cleanup has been done on failure.
+ *
+ * Return an appropriate hs_service_add_ephemeral_status_t. */
+hs_service_add_ephemeral_status_t
+hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports,
+                         int max_streams_per_rdv_circuit,
+                         int max_streams_close_circuit, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+  hs_service_t *service = NULL;
+
+  tor_assert(sk);
+  tor_assert(ports);
+  tor_assert(address_out);
+
+  service = hs_service_new(get_options());
+
+  /* Setup the service configuration with specifics. A default service is
+   * HS_VERSION_TWO so explicitly set it. */
+  service->config.version = HS_VERSION_THREE;
+  service->config.max_streams_per_rdv_circuit = max_streams_per_rdv_circuit;
+  service->config.max_streams_close_circuit = !!max_streams_close_circuit;
+  service->config.is_ephemeral = 1;
+  smartlist_free(service->config.ports);
+  service->config.ports = ports;
+
+  /* Handle the keys. */
+  memcpy(&service->keys.identity_sk, sk, sizeof(service->keys.identity_sk));
+  if (ed25519_public_key_generate(&service->keys.identity_pk,
+                                  &service->keys.identity_sk) < 0) {
+    log_warn(LD_CONFIG, "Unable to generate ed25519 public key"
+                        "for v3 service.");
+    ret = RSAE_BADPRIVKEY;
+    goto err;
+  }
+
+  /* Make sure we have at least one port. */
+  if (smartlist_len(service->config.ports) == 0) {
+    log_warn(LD_CONFIG, "At least one VIRTPORT/TARGET must be specified "
+                        "for v3 service.");
+    ret = RSAE_BADVIRTPORT;
+    goto err;
+  }
+
+  /* Build the onion address for logging purposes but also the control port
+   * uses it for the HS_DESC event. */
+  hs_build_address(&service->keys.identity_pk,
+                   (uint8_t) service->config.version,
+                   service->onion_address);
+
+  /* The only way the registration can fail is if the service public key
+   * already exists. */
+  if (BUG(register_service(hs_service_map, service) < 0)) {
+    log_warn(LD_CONFIG, "Onion Service private key collides with an "
+                        "existing v3 service.");
+    ret = RSAE_ADDREXISTS;
+    goto err;
+  }
+
+  log_info(LD_CONFIG, "Added ephemeral v3 onion service: %s",
+           safe_str_client(service->onion_address));
+
+  *address_out = tor_strdup(service->onion_address);
+  ret = RSAE_OKAY;
+  goto end;
+
+ err:
+  hs_service_free(service);
+
+ end:
+  memwipe(sk, 0, sizeof(ed25519_secret_key_t));
+  tor_free(sk);
+  return ret;
+}
+
+/* For the given onion address, delete the ephemeral service. Return 0 on
+ * success else -1 on error. */
+int
+hs_service_del_ephemeral(const char *address)
+{
+  uint8_t version;
+  ed25519_public_key_t pk;
+  hs_service_t *service = NULL;
+
+  tor_assert(address);
+
+  if (hs_parse_address(address, &pk, NULL, &version) < 0) {
+    log_warn(LD_CONFIG, "Requested malformed v3 onion address for removal.");
+    goto err;
+  }
+
+  if (version != HS_VERSION_THREE) {
+    log_warn(LD_CONFIG, "Requested version of onion address for removal "
+                        "is not supported.");
+    goto err;
+  }
+
+  service = find_service(hs_service_map, &pk);
+  if (service == NULL) {
+    log_warn(LD_CONFIG, "Requested non-existent v3 hidden service for "
+                        "removal.");
+    goto err;
+  }
+
+  if (!service->config.is_ephemeral) {
+    log_warn(LD_CONFIG, "Requested non-ephemeral v3 hidden service for "
+                        "removal.");
+    goto err;
+  }
+
+  /* Close circuits, remove from map and finally free. */
+  close_service_circuits(service);
+  remove_service(hs_service_map, service);
+  hs_service_free(service);
+
+  log_info(LD_CONFIG, "Removed ephemeral v3 hidden service: %s",
+           safe_str_client(address));
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Using the ed25519 public key pk, find a service for that key and return the
+ * current encoded descriptor as a newly allocated string or NULL if not
+ * found. This is used by the control port subsystem. */
+char *
+hs_service_lookup_current_desc(const ed25519_public_key_t *pk)
+{
+  const hs_service_t *service;
+
+  tor_assert(pk);
+
+  service = find_service(hs_service_map, pk);
+  if (service && service->desc_current) {
+    char *encoded_desc = NULL;
+    /* No matter what is the result (which should never be a failure), return
+     * the encoded variable, if success it will contain the right thing else
+     * it will be NULL. */
+    hs_desc_encode_descriptor(service->desc_current->desc,
+                              &service->desc_current->signing_kp,
+                              &encoded_desc);
+    return encoded_desc;
+  }
+
+  return NULL;
+}
+
+/* Return the number of service we have configured and usable. */
+unsigned int
+hs_service_get_num_services(void)
+{
+  if (hs_service_map == NULL) {
+    return 0;
+  }
+  return HT_SIZE(hs_service_map);
+}
+
+/* Called once an introduction circuit is closed. If the circuit doesn't have
+ * a v3 identifier, it is ignored. */
+void
+hs_service_intro_circ_has_closed(origin_circuit_t *circ)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_descriptor_t *desc = NULL;
+
+  tor_assert(circ);
+
+  if (circ->hs_ident == NULL) {
+    /* This is not a v3 circuit, ignore. */
+    goto end;
+  }
+
+  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
+  if (service == NULL) {
+    /* This is possible if the circuits are closed and the service is
+     * immediately deleted. */
+    log_info(LD_REND, "Unable to find any hidden service associated "
+                      "identity key %s on intro circuit %u.",
+             ed25519_fmt(&circ->hs_ident->identity_pk),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto end;
+  }
+  if (ip == NULL) {
+    /* The introduction point object has already been removed probably by our
+     * cleanup process so ignore. */
+    goto end;
+  }
+  /* Can't have an intro point object without a descriptor. */
+  tor_assert(desc);
+
+  /* Circuit disappeared so make sure the intro point is updated. By
+   * keeping the object in the descriptor, we'll be able to retry. */
+  ip->circuit_established = 0;
+
+ end:
+  return;
+}
+
+/* Given conn, a rendezvous edge connection acting as an exit stream, look up
+ * the hidden service for the circuit circ, and look up the port and address
+ * based on the connection port. Assign the actual connection address.
+ *
+ * Return 0 on success. Return -1 on failure and the caller should NOT close
+ * the circuit. Return -2 on failure and the caller MUST close the circuit for
+ * security reasons. */
+int
+hs_service_set_conn_addr_port(const origin_circuit_t *circ,
+                              edge_connection_t *conn)
+{
+  hs_service_t *service = NULL;
+
+  tor_assert(circ);
+  tor_assert(conn);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
+  tor_assert(circ->hs_ident);
+
+  get_objects_from_ident(circ->hs_ident, &service, NULL, NULL);
+
+  if (service == NULL) {
+    log_warn(LD_REND, "Unable to find any hidden service associated "
+                      "identity key %s on rendezvous circuit %u.",
+             ed25519_fmt(&circ->hs_ident->identity_pk),
+             TO_CIRCUIT(circ)->n_circ_id);
+    /* We want the caller to close the circuit because it's not a valid
+     * service so no danger. Attempting to bruteforce the entire key space by
+     * opening circuits to learn which service is being hosted here is
+     * impractical. */
+    goto err_close;
+  }
+
+  /* Enforce the streams-per-circuit limit, and refuse to provide a mapping if
+   * this circuit will exceed the limit. */
+  if (service->config.max_streams_per_rdv_circuit > 0 &&
+      (circ->hs_ident->num_rdv_streams >=
+       service->config.max_streams_per_rdv_circuit)) {
+#define MAX_STREAM_WARN_INTERVAL 600
+    static struct ratelim_t stream_ratelim =
+      RATELIM_INIT(MAX_STREAM_WARN_INTERVAL);
+    log_fn_ratelim(&stream_ratelim, LOG_WARN, LD_REND,
+                   "Maximum streams per circuit limit reached on "
+                   "rendezvous circuit %u for service %s. Circuit has "
+                   "%" PRIu64 " out of %" PRIu64 " streams. %s.",
+                   TO_CIRCUIT(circ)->n_circ_id,
+                   service->onion_address,
+                   circ->hs_ident->num_rdv_streams,
+                   service->config.max_streams_per_rdv_circuit,
+                   service->config.max_streams_close_circuit ?
+                    "Closing circuit" : "Ignoring open stream request");
+    if (service->config.max_streams_close_circuit) {
+      /* Service explicitly configured to close immediately. */
+      goto err_close;
+    }
+    /* Exceeding the limit makes tor silently ignore the stream creation
+     * request and keep the circuit open. */
+    goto err_no_close;
+  }
+
+  /* Find a virtual port of that service mathcing the one in the connection if
+   * successful, set the address in the connection. */
+  if (hs_set_conn_addr_port(service->config.ports, conn) < 0) {
+    log_info(LD_REND, "No virtual port mapping exists for port %d for "
+                      "hidden service %s.",
+             TO_CONN(conn)->port, service->onion_address);
+    if (service->config.allow_unknown_ports) {
+      /* Service explicitly allow connection to unknown ports so close right
+       * away because we do not care about port mapping. */
+      goto err_close;
+    }
+    /* If the service didn't explicitly allow it, we do NOT close the circuit
+     * here to raise the bar in terms of performance for port mapping. */
+    goto err_no_close;
+  }
+
+  /* Success. */
+  return 0;
+ err_close:
+  /* Indicate the caller that the circuit should be closed. */
+  return -2;
+ err_no_close:
+  /* Indicate the caller to NOT close the circuit. */
+  return -1;
+}
+
+/* Add to file_list every filename used by a configured hidden service, and to
+ * dir_list every directory path used by a configured hidden service. This is
+ * used by the sandbox subsystem to whitelist those. */
+void
+hs_service_lists_fnames_for_sandbox(smartlist_t *file_list,
+                                    smartlist_t *dir_list)
+{
+  tor_assert(file_list);
+  tor_assert(dir_list);
+
+  /* Add files and dirs for legacy services. */
+  rend_services_add_filenames_to_lists(file_list, dir_list);
+
+  /* Add files and dirs for v3+. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    /* Skip ephemeral service, they don't touch the disk. */
+    if (service->config.is_ephemeral) {
+      continue;
+    }
+    service_add_fnames_to_list(service, file_list);
+    smartlist_add_strdup(dir_list, service->config.directory_path);
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Called when our internal view of the directory has changed. We might have
+ * received a new batch of descriptors which might affect the shape of the
+ * HSDir hash ring. Signal that we should reexamine the hash ring and
+ * re-upload our HS descriptors if needed. */
+void
+hs_service_dir_info_changed(void)
+{
+  if (hs_service_get_num_services() > 0) {
+    /* New directory information usually goes every consensus so rate limit
+     * every 30 minutes to not be too conservative. */
+    static struct ratelim_t dir_info_changed_ratelim = RATELIM_INIT(30 * 60);
+    log_fn_ratelim(&dir_info_changed_ratelim, LOG_INFO, LD_REND,
+                   "New dirinfo arrived: consider reuploading descriptor");
+    consider_republishing_hs_descriptors = 1;
+  }
+}
+
+/* Called when we get an INTRODUCE2 cell on the circ. Respond to the cell and
+ * launch a circuit to the rendezvous point. */
+int
+hs_service_receive_introduce2(origin_circuit_t *circ, const uint8_t *payload,
+                              size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  /* Do some initial validation and logging before we parse the cell */
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_INTRO) {
+    log_warn(LD_PROTOCOL, "Received an INTRODUCE2 cell on a "
+                          "non introduction circuit of purpose %d",
+             TO_CIRCUIT(circ)->purpose);
+    goto done;
+  }
+
+  if (circ->hs_ident) {
+    ret = service_handle_introduce2(circ, payload, payload_len);
+    hs_stats_note_introduce2_cell(1);
+  } else {
+    ret = rend_service_receive_introduction(circ, payload, payload_len);
+    hs_stats_note_introduce2_cell(0);
+  }
+
+ done:
+  return ret;
+}
+
+/* Called when we get an INTRO_ESTABLISHED cell. Mark the circuit as an
+ * established introduction point. Return 0 on success else a negative value
+ * and the circuit is closed. */
+int
+hs_service_receive_intro_established(origin_circuit_t *circ,
+                                     const uint8_t *payload,
+                                     size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) {
+    log_warn(LD_PROTOCOL, "Received an INTRO_ESTABLISHED cell on a "
+                          "non introduction circuit of purpose %d",
+             TO_CIRCUIT(circ)->purpose);
+    goto err;
+  }
+
+  /* Handle both version. v2 uses rend_data and v3 uses the hs circuit
+   * identifier hs_ident. Can't be both. */
+  if (circ->hs_ident) {
+    ret = service_handle_intro_established(circ, payload, payload_len);
+  } else {
+    ret = rend_service_intro_established(circ, payload, payload_len);
+  }
+
+  if (ret < 0) {
+    goto err;
+  }
+  return 0;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Called when any kind of hidden service circuit is done building thus
+ * opened. This is the entry point from the circuit subsystem. */
+void
+hs_service_circuit_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Handle both version. v2 uses rend_data and v3 uses the hs circuit
+   * identifier hs_ident. Can't be both. */
+  switch (TO_CIRCUIT(circ)->purpose) {
+  case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
+    if (circ->hs_ident) {
+      service_intro_circ_has_opened(circ);
+    } else {
+      rend_service_intro_has_opened(circ);
+    }
+    break;
+  case CIRCUIT_PURPOSE_S_CONNECT_REND:
+    if (circ->hs_ident) {
+      service_rendezvous_circ_has_opened(circ);
+    } else {
+      rend_service_rendezvous_has_opened(circ);
+    }
+    break;
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Load and/or generate keys for all onion services including the client
+ * authorization if any. Return 0 on success, -1 on failure. */
+int
+hs_service_load_all_keys(void)
+{
+  /* Load v2 service keys if we have v2. */
+  if (rend_num_services() != 0) {
+    if (rend_service_load_all_keys(NULL) < 0) {
+      goto err;
+    }
+  }
+
+  /* Load or/and generate them for v3+. */
+  SMARTLIST_FOREACH_BEGIN(hs_service_staging_list, hs_service_t *, service) {
+    /* Ignore ephemeral service, they already have their keys set. */
+    if (service->config.is_ephemeral) {
+      continue;
+    }
+    log_info(LD_REND, "Loading v3 onion service keys from %s",
+             service_escaped_dir(service));
+    if (load_service_keys(service) < 0) {
+      goto err;
+    }
+    /* XXX: Load/Generate client authorization keys. (#20700) */
+  } SMARTLIST_FOREACH_END(service);
+
+  /* Final step, the staging list contains service in a quiescent state that
+   * is ready to be used. Register them to the global map. Once this is over,
+   * the staging list will be cleaned up. */
+  register_all_services();
+
+  /* All keys have been loaded successfully. */
+  return 0;
+ err:
+  return -1;
+}
+
+/* Put all service object in the given service list. After this, the caller
+ * looses ownership of every elements in the list and responsible to free the
+ * list pointer. */
+void
+hs_service_stage_services(const smartlist_t *service_list)
+{
+  tor_assert(service_list);
+  /* This list is freed at registration time but this function can be called
+   * multiple time. */
+  if (hs_service_staging_list == NULL) {
+    hs_service_staging_list = smartlist_new();
+  }
+  /* Add all service object to our staging list. Caller is responsible for
+   * freeing the service_list. */
+  smartlist_add_all(hs_service_staging_list, service_list);
+}
+
+/* Allocate and initilize a service object. The service configuration will
+ * contain the default values. Return the newly allocated object pointer. This
+ * function can't fail. */
+hs_service_t *
+hs_service_new(const or_options_t *options)
+{
+  hs_service_t *service = tor_malloc_zero(sizeof(hs_service_t));
+  /* Set default configuration value. */
+  set_service_default_config(&service->config, options);
+  /* Set the default service version. */
+  service->config.version = HS_SERVICE_DEFAULT_VERSION;
+  /* Allocate the CLIENT_PK replay cache in service state. */
+  service->state.replay_cache_rend_cookie =
+    replaycache_new(REND_REPLAY_TIME_INTERVAL, REND_REPLAY_TIME_INTERVAL);
+
+  return service;
+}
+
+/* Free the given <b>service</b> object and all its content. This function
+ * also takes care of wiping service keys from memory. It is safe to pass a
+ * NULL pointer. */
+void
+hs_service_free_(hs_service_t *service)
+{
+  if (service == NULL) {
+    return;
+  }
+
+  /* Free descriptors. Go over both descriptor with this loop. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    service_descriptor_free(desc);
+  } FOR_EACH_DESCRIPTOR_END;
+
+  /* Free service configuration. */
+  service_clear_config(&service->config);
+
+  /* Free replay cache from state. */
+  if (service->state.replay_cache_rend_cookie) {
+    replaycache_free(service->state.replay_cache_rend_cookie);
+  }
+
+  /* Wipe service keys. */
+  memwipe(&service->keys.identity_sk, 0, sizeof(service->keys.identity_sk));
+
+  tor_free(service);
+}
+
+/* Periodic callback. Entry point from the main loop to the HS service
+ * subsystem. This is call every second. This is skipped if tor can't build a
+ * circuit or the network is disabled. */
+void
+hs_service_run_scheduled_events(time_t now)
+{
+  /* First thing we'll do here is to make sure our services are in a
+   * quiescent state for the scheduled events. */
+  run_housekeeping_event(now);
+
+  /* Order matters here. We first make sure the descriptor object for each
+   * service contains the latest data. Once done, we check if we need to open
+   * new introduction circuit. Finally, we try to upload the descriptor for
+   * each service. */
+
+  /* Make sure descriptors are up to date. */
+  run_build_descriptor_event(now);
+  /* Make sure services have enough circuits. */
+  run_build_circuit_event(now);
+  /* Upload the descriptors if needed/possible. */
+  run_upload_descriptor_event(now);
+}
+
+/* Initialize the service HS subsystem. */
+void
+hs_service_init(void)
+{
+  /* Should never be called twice. */
+  tor_assert(!hs_service_map);
+  tor_assert(!hs_service_staging_list);
+
+  /* v2 specific. */
+  rend_service_init();
+
+  hs_service_map = tor_malloc_zero(sizeof(struct hs_service_ht));
+  HT_INIT(hs_service_ht, hs_service_map);
+
+  hs_service_staging_list = smartlist_new();
+}
+
+/* Release all global storage of the hidden service subsystem. */
+void
+hs_service_free_all(void)
+{
+  rend_service_free_all();
+  service_free_all();
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/* Return the global service map size. Only used by unit test. */
+STATIC unsigned int
+get_hs_service_map_size(void)
+{
+  return HT_SIZE(hs_service_map);
+}
+
+/* Return the staging list size. Only used by unit test. */
+STATIC int
+get_hs_service_staging_list_size(void)
+{
+  return smartlist_len(hs_service_staging_list);
+}
+
+STATIC hs_service_ht *
+get_hs_service_map(void)
+{
+  return hs_service_map;
+}
+
+STATIC hs_service_t *
+get_first_service(void)
+{
+  hs_service_t **obj = HT_START(hs_service_ht, hs_service_map);
+  if (obj == NULL) {
+    return NULL;
+  }
+  return *obj;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h
new file mode 100644
index 0000000000..4676042b54
--- /dev/null
+++ b/src/feature/hs/hs_service.h
@@ -0,0 +1,378 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_service.h
+ * \brief Header file containing service data for the HS subsytem.
+ **/
+
+#ifndef TOR_HS_SERVICE_H
+#define TOR_HS_SERVICE_H
+
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "or/replaycache.h"
+
+#include "or/hs_common.h"
+#include "or/hs_descriptor.h"
+#include "or/hs_ident.h"
+#include "or/hs_intropoint.h"
+
+/* Trunnel */
+#include "trunnel/hs/cell_establish_intro.h"
+
+/* When loading and configuring a service, this is the default version it will
+ * be configured for as it is possible that no HiddenServiceVersion is
+ * present. */
+#define HS_SERVICE_DEFAULT_VERSION HS_VERSION_TWO
+
+/* As described in the specification, service publishes their next descriptor
+ * at a random time between those two values (in seconds). */
+#define HS_SERVICE_NEXT_UPLOAD_TIME_MIN (60 * 60)
+#define HS_SERVICE_NEXT_UPLOAD_TIME_MAX (120 * 60)
+
+/* Service side introduction point. */
+typedef struct hs_service_intro_point_t {
+  /* Top level intropoint "shared" data between client/service. */
+  hs_intropoint_t base;
+
+  /* Onion key of the introduction point used to extend to it for the ntor
+   * handshake. */
+  curve25519_public_key_t onion_key;
+
+  /* Authentication keypair used to create the authentication certificate
+   * which is published in the descriptor. */
+  ed25519_keypair_t auth_key_kp;
+
+  /* Encryption keypair for the "ntor" type. */
+  curve25519_keypair_t enc_key_kp;
+
+  /* Legacy key if that intro point doesn't support v3. This should be used if
+   * the base object legacy flag is set. */
+  crypto_pk_t *legacy_key;
+  /* Legacy key SHA1 public key digest. This should be used only if the base
+   * object legacy flag is set. */
+  uint8_t legacy_key_digest[DIGEST_LEN];
+
+  /* Amount of INTRODUCE2 cell accepted from this intro point. */
+  uint64_t introduce2_count;
+
+  /* Maximum number of INTRODUCE2 cell this intro point should accept. */
+  uint64_t introduce2_max;
+
+  /* The time at which this intro point should expire and stop being used. */
+  time_t time_to_expire;
+
+  /* The amount of circuit creation we've made to this intro point. This is
+   * incremented every time we do a circuit relaunch on this intro point which
+   * is triggered when the circuit dies but the node is still in the
+   * consensus. After MAX_INTRO_POINT_CIRCUIT_RETRIES, we give up on it. */
+  uint32_t circuit_retries;
+
+  /* Set if this intro point has an established circuit. */
+  unsigned int circuit_established : 1;
+
+  /* Replay cache recording the encrypted part of an INTRODUCE2 cell that the
+   * circuit associated with this intro point has received. This is used to
+   * prevent replay attacks. */
+  replaycache_t *replay_cache;
+} hs_service_intro_point_t;
+
+/* Object handling introduction points of a service. */
+typedef struct hs_service_intropoints_t {
+  /* The time at which we've started our retry period to build circuits. We
+   * don't want to stress circuit creation so we can only retry for a certain
+   * time and then after we stop and wait. */
+  time_t retry_period_started;
+
+  /* Number of circuit we've launched during a single retry period. */
+  unsigned int num_circuits_launched;
+
+  /* Contains the current hs_service_intro_point_t objects indexed by
+   * authentication public key. */
+  digest256map_t *map;
+
+  /* Contains node's identity key digest that were introduction point for this
+   * descriptor but were retried to many times. We keep those so we avoid
+   * re-picking them over and over for a circuit retry period.
+   * XXX: Once we have #22173, change this to only use ed25519 identity. */
+  digestmap_t *failed_id;
+} hs_service_intropoints_t;
+
+/* Representation of a service descriptor. */
+typedef struct hs_service_descriptor_t {
+  /* Decoded descriptor. This object is used for encoding when the service
+   * publishes the descriptor. */
+  hs_descriptor_t *desc;
+
+  /* Descriptor signing keypair. */
+  ed25519_keypair_t signing_kp;
+
+  /* Blinded keypair derived from the master identity public key. */
+  ed25519_keypair_t blinded_kp;
+
+  /* When is the next time when we should upload the descriptor. */
+  time_t next_upload_time;
+
+  /* Introduction points assign to this descriptor which contains
+   * hs_service_intropoints_t object indexed by authentication key (the RSA
+   * key if the node is legacy). */
+  hs_service_intropoints_t intro_points;
+
+  /* The time period number this descriptor has been created for. */
+  uint64_t time_period_num;
+
+  /* True iff we have missing intro points for this descriptor because we
+   * couldn't pick any nodes. */
+  unsigned int missing_intro_points : 1;
+
+  /** List of the responsible HSDirs (their b64ed identity digest) last time we
+   *  uploaded this descriptor. If the set of responsible HSDirs is different
+   *  from this list, this means we received new dirinfo and we need to
+   *  reupload our descriptor. */
+  smartlist_t *previous_hsdirs;
+} hs_service_descriptor_t;
+
+/* Service key material. */
+typedef struct hs_service_keys_t {
+  /* Master identify public key. */
+  ed25519_public_key_t identity_pk;
+  /* Master identity private key. */
+  ed25519_secret_key_t identity_sk;
+  /* True iff the key is kept offline which means the identity_sk MUST not be
+   * used in that case. */
+  unsigned int is_identify_key_offline : 1;
+} hs_service_keys_t;
+
+/* Service configuration. The following are set from the torrc options either
+ * set by the configuration file or by the control port. Nothing else should
+ * change those values. */
+typedef struct hs_service_config_t {
+  /* Protocol version of the service. Specified by HiddenServiceVersion
+   * option. */
+  uint32_t version;
+
+  /* List of rend_service_port_config_t */
+  smartlist_t *ports;
+
+  /* Path on the filesystem where the service persistent data is stored. NULL
+   * if the service is ephemeral. Specified by HiddenServiceDir option. */
+  char *directory_path;
+
+  /* The maximum number of simultaneous streams per rendezvous circuit that
+   * are allowed to be created. No limit if 0. Specified by
+   * HiddenServiceMaxStreams option. */
+  uint64_t max_streams_per_rdv_circuit;
+
+  /* If true, we close circuits that exceed the max_streams_per_rdv_circuit
+   * limit. Specified by HiddenServiceMaxStreamsCloseCircuit option. */
+  unsigned int max_streams_close_circuit : 1;
+
+  /* How many introduction points this service has. Specified by
+   * HiddenServiceNumIntroductionPoints option. */
+  unsigned int num_intro_points;
+
+  /* True iff we allow request made on unknown ports. Specified by
+   * HiddenServiceAllowUnknownPorts option. */
+  unsigned int allow_unknown_ports : 1;
+
+  /* If true, this service is a Single Onion Service. Specified by
+   * HiddenServiceSingleHopMode and HiddenServiceNonAnonymousMode options. */
+  unsigned int is_single_onion : 1;
+
+  /* If true, allow group read permissions on the directory_path. Specified by
+   * HiddenServiceDirGroupReadable option. */
+  unsigned int dir_group_readable : 1;
+
+  /* Is this service ephemeral? */
+  unsigned int is_ephemeral : 1;
+} hs_service_config_t;
+
+/* Service state. */
+typedef struct hs_service_state_t {
+  /* The time at which we've started our retry period to build circuits. We
+   * don't want to stress circuit creation so we can only retry for a certain
+   * time and then after we stop and wait. */
+  time_t intro_circ_retry_started_time;
+
+  /* Number of circuit we've launched during a single retry period. This
+   * should never go over MAX_INTRO_CIRCS_PER_PERIOD. */
+  unsigned int num_intro_circ_launched;
+
+  /* Replay cache tracking the REND_COOKIE found in INTRODUCE2 cell to detect
+   * repeats. Clients may send INTRODUCE1 cells for the same rendezvous point
+   * through two or more different introduction points; when they do, this
+   * keeps us from launching multiple simultaneous attempts to connect to the
+   * same rend point. */
+  replaycache_t *replay_cache_rend_cookie;
+
+  /* When is the next time we should rotate our descriptors. This is has to be
+   * done at the start time of the next SRV protocol run. */
+  time_t next_rotation_time;
+} hs_service_state_t;
+
+/* Representation of a service running on this tor instance. */
+typedef struct hs_service_t {
+  /* Onion address base32 encoded and NUL terminated. We keep it for logging
+   * purposes so we don't have to build it everytime. */
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  /* Hashtable node: use to look up the service by its master public identity
+   * key in the service global map. */
+  HT_ENTRY(hs_service_t) hs_service_node;
+
+  /* Service state which contains various flags and counters. */
+  hs_service_state_t state;
+
+  /* Key material of the service. */
+  hs_service_keys_t keys;
+
+  /* Configuration of the service. */
+  hs_service_config_t config;
+
+  /* Current descriptor. */
+  hs_service_descriptor_t *desc_current;
+  /* Next descriptor. */
+  hs_service_descriptor_t *desc_next;
+
+  /* XXX: Credential (client auth.) #20700. */
+
+} hs_service_t;
+
+/* For the service global hash map, we define a specific type for it which
+ * will make it safe to use and specific to some controlled parameters such as
+ * the hashing function and how to compare services. */
+typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht;
+
+/* API */
+
+/* Global initializer and cleanup function. */
+void hs_service_init(void);
+void hs_service_free_all(void);
+
+/* Service new/free functions. */
+hs_service_t *hs_service_new(const or_options_t *options);
+void hs_service_free_(hs_service_t *service);
+#define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s))
+
+unsigned int hs_service_get_num_services(void);
+void hs_service_stage_services(const smartlist_t *service_list);
+int hs_service_load_all_keys(void);
+void hs_service_lists_fnames_for_sandbox(smartlist_t *file_list,
+                                         smartlist_t *dir_list);
+int hs_service_set_conn_addr_port(const origin_circuit_t *circ,
+                                  edge_connection_t *conn);
+
+void hs_service_map_has_changed(void);
+void hs_service_dir_info_changed(void);
+void hs_service_run_scheduled_events(time_t now);
+void hs_service_circuit_has_opened(origin_circuit_t *circ);
+int hs_service_receive_intro_established(origin_circuit_t *circ,
+                                         const uint8_t *payload,
+                                         size_t payload_len);
+int hs_service_receive_introduce2(origin_circuit_t *circ,
+                                  const uint8_t *payload,
+                                  size_t payload_len);
+
+void hs_service_intro_circ_has_closed(origin_circuit_t *circ);
+
+char *hs_service_lookup_current_desc(const ed25519_public_key_t *pk);
+
+hs_service_add_ephemeral_status_t
+hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports,
+                         int max_streams_per_rdv_circuit,
+                         int max_streams_close_circuit, char **address_out);
+int hs_service_del_ephemeral(const char *address);
+
+/* Used outside of the HS subsystem by the control port command HSPOST. */
+void hs_service_upload_desc_to_dir(const char *encoded_desc,
+                                   const uint8_t version,
+                                   const ed25519_public_key_t *identity_pk,
+                                   const ed25519_public_key_t *blinded_pk,
+                                   const routerstatus_t *hsdir_rs);
+
+#ifdef HS_SERVICE_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+/* Useful getters for unit tests. */
+STATIC unsigned int get_hs_service_map_size(void);
+STATIC int get_hs_service_staging_list_size(void);
+STATIC hs_service_ht *get_hs_service_map(void);
+STATIC hs_service_t *get_first_service(void);
+STATIC hs_service_intro_point_t *service_intro_point_find_by_ident(
+                                         const hs_service_t *service,
+                                         const hs_ident_circuit_t *ident);
+#endif
+
+/* Service accessors. */
+STATIC hs_service_t *find_service(hs_service_ht *map,
+                                  const ed25519_public_key_t *pk);
+STATIC void remove_service(hs_service_ht *map, hs_service_t *service);
+STATIC int register_service(hs_service_ht *map, hs_service_t *service);
+/* Service introduction point functions. */
+STATIC hs_service_intro_point_t *service_intro_point_new(
+                                         const extend_info_t *ei,
+                                         unsigned int is_legacy);
+STATIC void service_intro_point_free_(hs_service_intro_point_t *ip);
+#define service_intro_point_free(ip)                            \
+  FREE_AND_NULL(hs_service_intro_point_t,             \
+                          service_intro_point_free_, (ip))
+STATIC void service_intro_point_add(digest256map_t *map,
+                                    hs_service_intro_point_t *ip);
+STATIC void service_intro_point_remove(const hs_service_t *service,
+                                       const hs_service_intro_point_t *ip);
+STATIC hs_service_intro_point_t *service_intro_point_find(
+                                 const hs_service_t *service,
+                                 const ed25519_public_key_t *auth_key);
+/* Service descriptor functions. */
+STATIC hs_service_descriptor_t *service_descriptor_new(void);
+STATIC hs_service_descriptor_t *service_desc_find_by_intro(
+                                         const hs_service_t *service,
+                                         const hs_service_intro_point_t *ip);
+/* Helper functions. */
+STATIC void get_objects_from_ident(const hs_ident_circuit_t *ident,
+                                   hs_service_t **service,
+                                   hs_service_intro_point_t **ip,
+                                   hs_service_descriptor_t **desc);
+STATIC const node_t *
+get_node_from_intro_point(const hs_service_intro_point_t *ip);
+STATIC int can_service_launch_intro_circuit(hs_service_t *service,
+                                            time_t now);
+STATIC int intro_point_should_expire(const hs_service_intro_point_t *ip,
+                                     time_t now);
+STATIC void run_housekeeping_event(time_t now);
+STATIC void rotate_all_descriptors(time_t now);
+STATIC void build_all_descriptors(time_t now);
+STATIC void update_all_descriptors(time_t now);
+STATIC void run_upload_descriptor_event(time_t now);
+
+STATIC char *
+encode_desc_rev_counter_for_state(const hs_service_descriptor_t *desc);
+
+STATIC void service_descriptor_free_(hs_service_descriptor_t *desc);
+#define service_descriptor_free(d) \
+  FREE_AND_NULL(hs_service_descriptor_t, \
+                           service_descriptor_free_, (d))
+
+STATIC uint64_t
+check_state_line_for_service_rev_counter(const char *state_line,
+                                    const ed25519_public_key_t *blinded_pubkey,
+                                    int *service_found_out);
+
+STATIC int
+write_address_to_file(const hs_service_t *service, const char *fname_);
+
+STATIC void upload_descriptor_to_all(const hs_service_t *service,
+                                     hs_service_descriptor_t *desc);
+
+STATIC void service_desc_schedule_upload(hs_service_descriptor_t *desc,
+                                         time_t now,
+                                         int descriptor_changed);
+
+STATIC int service_desc_hsdirs_changed(const hs_service_t *service,
+                                const hs_service_descriptor_t *desc);
+
+#endif /* defined(HS_SERVICE_PRIVATE) */
+
+#endif /* !defined(TOR_HS_SERVICE_H) */
+
diff --git a/src/feature/hs/hs_stats.c b/src/feature/hs/hs_stats.c
new file mode 100644
index 0000000000..c8a99b19d4
--- /dev/null
+++ b/src/feature/hs/hs_stats.c
@@ -0,0 +1,58 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_stats.c
+ * \brief Keeps stats about the activity of our onion service(s).
+ **/
+
+#include "or/or.h"
+#include "or/hs_stats.h"
+#include "or/hs_service.h"
+
+/** Number of v3 INTRODUCE2 cells received */
+static uint32_t n_introduce2_v3 = 0;
+/** Number of v2 INTRODUCE2 cells received */
+static uint32_t n_introduce2_v2 = 0;
+/** Number of attempts to make a circuit to a rendezvous point */
+static uint32_t n_rendezvous_launches = 0;
+
+/** Note that we received another INTRODUCE2 cell. */
+void
+hs_stats_note_introduce2_cell(int is_hsv3)
+{
+  if (is_hsv3) {
+    n_introduce2_v3++;
+  } else {
+    n_introduce2_v2++;
+  }
+}
+
+/** Return the number of v3 INTRODUCE2 cells we have received. */
+uint32_t
+hs_stats_get_n_introduce2_v3_cells(void)
+{
+  return n_introduce2_v3;
+}
+
+/** Return the number of v2 INTRODUCE2 cells we have received. */
+uint32_t
+hs_stats_get_n_introduce2_v2_cells(void)
+{
+  return n_introduce2_v2;
+}
+
+/** Note that we attempted to launch another circuit to a rendezvous point. */
+void
+hs_stats_note_service_rendezvous_launch(void)
+{
+  n_rendezvous_launches++;
+}
+
+/** Return the number of rendezvous circuits we have attempted to launch. */
+uint32_t
+hs_stats_get_n_rendezvous_launches(void)
+{
+  return n_rendezvous_launches;
+}
+
diff --git a/src/feature/hs/hs_stats.h b/src/feature/hs/hs_stats.h
new file mode 100644
index 0000000000..a946ad75e5
--- /dev/null
+++ b/src/feature/hs/hs_stats.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_stats.h
+ * \brief Header file for hs_stats.c
+ **/
+
+void hs_stats_note_introduce2_cell(int is_hsv3);
+uint32_t hs_stats_get_n_introduce2_v3_cells(void);
+uint32_t hs_stats_get_n_introduce2_v2_cells(void);
+void hs_stats_note_service_rendezvous_launch(void);
+uint32_t hs_stats_get_n_rendezvous_launches(void);
+
diff --git a/src/feature/hs/hsdir_index_st.h b/src/feature/hs/hsdir_index_st.h
new file mode 100644
index 0000000000..de5cc9bd16
--- /dev/null
+++ b/src/feature/hs/hsdir_index_st.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef HSDIR_INDEX_ST_H
+#define HSDIR_INDEX_ST_H
+
+/* Hidden service directory index used in a node_t which is set once we set
+ * the consensus. */
+struct hsdir_index_t {
+  /* HSDir index to use when fetching a descriptor. */
+  uint8_t fetch[DIGEST256_LEN];
+
+  /* HSDir index used by services to store their first and second
+   * descriptor. The first descriptor is chronologically older than the second
+   * one and uses older TP and SRV values. */
+  uint8_t store_first[DIGEST256_LEN];
+  uint8_t store_second[DIGEST256_LEN];
+};
+
+#endif
+
diff --git a/src/feature/hs_common/replaycache.c b/src/feature/hs_common/replaycache.c
new file mode 100644
index 0000000000..b5cc6a2823
--- /dev/null
+++ b/src/feature/hs_common/replaycache.c
@@ -0,0 +1,209 @@
+ /* Copyright (c) 2012-2018, The Tor Project, Inc. */
+ /* See LICENSE for licensing information */
+
+/**
+ * \file replaycache.c
+ *
+ * \brief Self-scrubbing replay cache for rendservice.c
+ *
+ * To prevent replay attacks, hidden services need to recognize INTRODUCE2
+ * cells that they've already seen, and drop them.  If they didn't, then
+ * sending the same INTRODUCE2 cell over and over would force the hidden
+ * service to make a huge number of circuits to the same rendezvous
+ * point, aiding traffic analysis.
+ *
+ * (It's not that simple, actually.  We only check for replays in the
+ * RSA-encrypted portion of the handshake, since the rest of the handshake is
+ * malleable.)
+ *
+ * This module is used from rendservice.c.
+ */
+
+#define REPLAYCACHE_PRIVATE
+
+#include "or/or.h"
+#include "or/replaycache.h"
+
+/** Free the replaycache r and all of its entries.
+ */
+void
+replaycache_free_(replaycache_t *r)
+{
+  if (!r) {
+    log_info(LD_BUG, "replaycache_free() called on NULL");
+    return;
+  }
+
+  if (r->digests_seen) digest256map_free(r->digests_seen, tor_free_);
+
+  tor_free(r);
+}
+
+/** Allocate a new, empty replay detection cache, where horizon is the time
+ * for entries to age out and interval is the time after which the cache
+ * should be scrubbed for old entries.
+ */
+replaycache_t *
+replaycache_new(time_t horizon, time_t interval)
+{
+  replaycache_t *r = NULL;
+
+  if (horizon < 0) {
+    log_info(LD_BUG, "replaycache_new() called with negative"
+        " horizon parameter");
+    goto err;
+  }
+
+  if (interval < 0) {
+    log_info(LD_BUG, "replaycache_new() called with negative interval"
+        " parameter");
+    interval = 0;
+  }
+
+  r = tor_malloc(sizeof(*r));
+  r->scrub_interval = interval;
+  r->scrubbed = 0;
+  r->horizon = horizon;
+  r->digests_seen = digest256map_new();
+
+ err:
+  return r;
+}
+
+/** See documentation for replaycache_add_and_test().
+ */
+STATIC int
+replaycache_add_and_test_internal(
+    time_t present, replaycache_t *r, const void *data, size_t len,
+    time_t *elapsed)
+{
+  int rv = 0;
+  uint8_t digest[DIGEST256_LEN];
+  time_t *access_time;
+
+  /* sanity check */
+  if (present <= 0 || !r || !data || len == 0) {
+    log_info(LD_BUG, "replaycache_add_and_test_internal() called with stupid"
+        " parameters; please fix this.");
+    goto done;
+  }
+
+  /* compute digest */
+  crypto_digest256((char *)digest, (const char *)data, len, DIGEST_SHA256);
+
+  /* check map */
+  access_time = digest256map_get(r->digests_seen, digest);
+
+  /* seen before? */
+  if (access_time != NULL) {
+    /*
+     * If it's far enough in the past, no hit.  If the horizon is zero, we
+     * never expire.
+     */
+    if (*access_time >= present - r->horizon || r->horizon == 0) {
+      /* replay cache hit, return 1 */
+      rv = 1;
+      /* If we want to output an elapsed time, do so */
+      if (elapsed) {
+        if (present >= *access_time) {
+          *elapsed = present - *access_time;
+        } else {
+          /* We shouldn't really be seeing hits from the future, but... */
+          *elapsed = 0;
+        }
+      }
+    }
+    /*
+     * If it's ahead of the cached time, update
+     */
+    if (*access_time < present) {
+      *access_time = present;
+    }
+  } else {
+    /* No, so no hit and update the digest map with the current time */
+    access_time = tor_malloc(sizeof(*access_time));
+    *access_time = present;
+    digest256map_set(r->digests_seen, digest, access_time);
+  }
+
+  /* now scrub the cache if it's time */
+  replaycache_scrub_if_needed_internal(present, r);
+
+ done:
+  return rv;
+}
+
+/** See documentation for replaycache_scrub_if_needed().
+ */
+STATIC void
+replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r)
+{
+  digest256map_iter_t *itr = NULL;
+  const uint8_t *digest;
+  void *valp;
+  time_t *access_time;
+
+  /* sanity check */
+  if (!r || !(r->digests_seen)) {
+    log_info(LD_BUG, "replaycache_scrub_if_needed_internal() called with"
+        " stupid parameters; please fix this.");
+    return;
+  }
+
+  /* scrub time yet? (scrubbed == 0 indicates never scrubbed before) */
+  if (present - r->scrubbed < r->scrub_interval && r->scrubbed > 0) return;
+
+  /* if we're never expiring, don't bother scrubbing */
+  if (r->horizon == 0) return;
+
+  /* okay, scrub time */
+  itr = digest256map_iter_init(r->digests_seen);
+  while (!digest256map_iter_done(itr)) {
+    digest256map_iter_get(itr, &digest, &valp);
+    access_time = (time_t *)valp;
+    /* aged out yet? */
+    if (*access_time < present - r->horizon) {
+      /* Advance the iterator and remove this one */
+      itr = digest256map_iter_next_rmv(r->digests_seen, itr);
+      /* Free the value removed */
+      tor_free(access_time);
+    } else {
+      /* Just advance the iterator */
+      itr = digest256map_iter_next(r->digests_seen, itr);
+    }
+  }
+
+  /* update scrubbed timestamp */
+  if (present > r->scrubbed) r->scrubbed = present;
+}
+
+/** Test the buffer of length len point to by data against the replay cache r;
+ * the digest of the buffer will be added to the cache at the current time,
+ * and the function will return 1 if it was already seen within the cache's
+ * horizon, or 0 otherwise.
+ */
+int
+replaycache_add_and_test(replaycache_t *r, const void *data, size_t len)
+{
+  return replaycache_add_and_test_internal(time(NULL), r, data, len, NULL);
+}
+
+/** Like replaycache_add_and_test(), but if it's a hit also return the time
+ * elapsed since this digest was last seen.
+ */
+int
+replaycache_add_test_and_elapsed(
+    replaycache_t *r, const void *data, size_t len, time_t *elapsed)
+{
+  return replaycache_add_and_test_internal(time(NULL), r, data, len, elapsed);
+}
+
+/** Scrub aged entries out of r if sufficiently long has elapsed since r was
+ * last scrubbed.
+ */
+void
+replaycache_scrub_if_needed(replaycache_t *r)
+{
+  replaycache_scrub_if_needed_internal(time(NULL), r);
+}
+
diff --git a/src/feature/hs_common/replaycache.h b/src/feature/hs_common/replaycache.h
new file mode 100644
index 0000000000..3118a88a1a
--- /dev/null
+++ b/src/feature/hs_common/replaycache.h
@@ -0,0 +1,67 @@
+/* Copyright (c) 2012-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file replaycache.h
+ * \brief Header file for replaycache.c.
+ **/
+
+#ifndef TOR_REPLAYCACHE_H
+#define TOR_REPLAYCACHE_H
+
+typedef struct replaycache_t replaycache_t;
+
+#ifdef REPLAYCACHE_PRIVATE
+
+struct replaycache_t {
+  /* Scrub interval */
+  time_t scrub_interval;
+  /* Last scrubbed */
+  time_t scrubbed;
+  /*
+   * Horizon
+   * (don't return true on digests in the cache but older than this)
+   */
+  time_t horizon;
+  /*
+   * Digest map: keys are digests, values are times the digest was last seen
+   */
+  digest256map_t *digests_seen;
+};
+
+#endif /* defined(REPLAYCACHE_PRIVATE) */
+
+/* replaycache_t free/new */
+
+void replaycache_free_(replaycache_t *r);
+#define replaycache_free(r) \
+  FREE_AND_NULL(replaycache_t, replaycache_free_, (r))
+replaycache_t * replaycache_new(time_t horizon, time_t interval);
+
+#ifdef REPLAYCACHE_PRIVATE
+
+/*
+ * replaycache_t internal functions:
+ *
+ * These take the time to treat as the present as an argument for easy unit
+ * testing.  For everything else, use the wrappers below instead.
+ */
+
+STATIC int replaycache_add_and_test_internal(
+    time_t present, replaycache_t *r, const void *data, size_t len,
+    time_t *elapsed);
+STATIC void replaycache_scrub_if_needed_internal(
+    time_t present, replaycache_t *r);
+
+#endif /* defined(REPLAYCACHE_PRIVATE) */
+
+/*
+ * replaycache_t methods
+ */
+
+int replaycache_add_and_test(replaycache_t *r, const void *data, size_t len);
+int replaycache_add_test_and_elapsed(
+    replaycache_t *r, const void *data, size_t len, time_t *elapsed);
+void replaycache_scrub_if_needed(replaycache_t *r);
+
+#endif /* !defined(TOR_REPLAYCACHE_H) */
diff --git a/src/feature/hs_common/shared_random_client.c b/src/feature/hs_common/shared_random_client.c
new file mode 100644
index 0000000000..42a5b42f60
--- /dev/null
+++ b/src/feature/hs_common/shared_random_client.c
@@ -0,0 +1,260 @@
+/* Copyright (c) 2018-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shared_random_client.c
+ * \brief This file contains functions that are from the shared random
+ *        subsystem but used by many part of tor. The full feature is built
+ *        as part of the dirauth module.
+ **/
+
+#define SHARED_RANDOM_CLIENT_PRIVATE
+#include "or/shared_random_client.h"
+
+#include "or/config.h"
+#include "or/voting_schedule.h"
+#include "or/networkstatus.h"
+#include "lib/encoding/binascii.h"
+
+#include "or/networkstatus_st.h"
+
+/* Convert a given srv object to a string for the control port. This doesn't
+ * fail and the srv object MUST be valid. */
+static char *
+srv_to_control_string(const sr_srv_t *srv)
+{
+  char *srv_str;
+  char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1];
+  tor_assert(srv);
+
+  sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv);
+  tor_asprintf(&srv_str, "%s", srv_hash_encoded);
+  return srv_str;
+}
+
+/* Return the voting interval of the tor vote subsystem. */
+int
+get_voting_interval(void)
+{
+  int interval;
+  networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL));
+
+  if (consensus) {
+    interval = (int)(consensus->fresh_until - consensus->valid_after);
+  } else {
+    /* Same for both a testing and real network. We voluntarily ignore the
+     * InitialVotingInterval since it complexifies things and it doesn't
+     * affect the SR protocol. */
+    interval = get_options()->V3AuthVotingInterval;
+  }
+  tor_assert(interval > 0);
+  return interval;
+}
+
+/* Given the time <b>now</b>, return the start time of the current round of
+ * the SR protocol. For example, if it's 23:47:08, the current round thus
+ * started at 23:47:00 for a voting interval of 10 seconds. */
+time_t
+get_start_time_of_current_round(void)
+{
+  const or_options_t *options = get_options();
+  int voting_interval = get_voting_interval();
+  /* First, get the start time of the next round */
+  time_t next_start = voting_schedule_get_next_valid_after_time();
+  /* Now roll back next_start by a voting interval to find the start time of
+     the current round. */
+  time_t curr_start = voting_schedule_get_start_of_next_interval(
+                                     next_start - voting_interval - 1,
+                                     voting_interval,
+                                     options->TestingV3AuthVotingStartOffset);
+  return curr_start;
+}
+
+/*
+ * Public API
+ */
+
+/* Encode the given shared random value and put it in dst. Destination
+ * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */
+void
+sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv)
+{
+  int ret;
+  /* Extra byte for the NULL terminated char. */
+  char buf[SR_SRV_VALUE_BASE64_LEN + 1];
+
+  tor_assert(dst);
+  tor_assert(srv);
+  tor_assert(dst_len >= sizeof(buf));
+
+  ret = base64_encode(buf, sizeof(buf), (const char *) srv->value,
+                      sizeof(srv->value), 0);
+  /* Always expect the full length without the NULL byte. */
+  tor_assert(ret == (sizeof(buf) - 1));
+  tor_assert(ret <= (int) dst_len);
+  strlcpy(dst, buf, dst_len);
+}
+
+/* Return the current SRV string representation for the control port. Return a
+ * newly allocated string on success containing the value else "" if not found
+ * or if we don't have a valid consensus yet. */
+char *
+sr_get_current_for_control(void)
+{
+  char *srv_str;
+  const networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && c->sr_info.current_srv) {
+    srv_str = srv_to_control_string(c->sr_info.current_srv);
+  } else {
+    srv_str = tor_strdup("");
+  }
+  return srv_str;
+}
+
+/* Return the previous SRV string representation for the control port. Return
+ * a newly allocated string on success containing the value else "" if not
+ * found or if we don't have a valid consensus yet. */
+char *
+sr_get_previous_for_control(void)
+{
+  char *srv_str;
+  const networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && c->sr_info.previous_srv) {
+    srv_str = srv_to_control_string(c->sr_info.previous_srv);
+  } else {
+    srv_str = tor_strdup("");
+  }
+  return srv_str;
+}
+
+/* Return current shared random value from the latest consensus. Caller can
+ * NOT keep a reference to the returned pointer. Return NULL if none. */
+const sr_srv_t *
+sr_get_current(const networkstatus_t *ns)
+{
+  const networkstatus_t *consensus;
+
+  /* Use provided ns else get a live one */
+  if (ns) {
+    consensus = ns;
+  } else {
+    consensus = networkstatus_get_live_consensus(approx_time());
+  }
+  /* Ideally we would never be asked for an SRV without a live consensus. Make
+   * sure this assumption is correct. */
+  tor_assert_nonfatal(consensus);
+
+  if (consensus) {
+    return consensus->sr_info.current_srv;
+  }
+  return NULL;
+}
+
+/* Return previous shared random value from the latest consensus. Caller can
+ * NOT keep a reference to the returned pointer. Return NULL if none. */
+const sr_srv_t *
+sr_get_previous(const networkstatus_t *ns)
+{
+  const networkstatus_t *consensus;
+
+  /* Use provided ns else get a live one */
+  if (ns) {
+    consensus = ns;
+  } else {
+    consensus = networkstatus_get_live_consensus(approx_time());
+  }
+  /* Ideally we would never be asked for an SRV without a live consensus. Make
+   * sure this assumption is correct. */
+  tor_assert_nonfatal(consensus);
+
+  if (consensus) {
+    return consensus->sr_info.previous_srv;
+  }
+  return NULL;
+}
+
+/* Parse a list of arguments from a SRV value either from a vote, consensus
+ * or from our disk state and return a newly allocated srv object. NULL is
+ * returned on error.
+ *
+ * The arguments' order:
+ *    num_reveals, value
+ */
+sr_srv_t *
+sr_parse_srv(const smartlist_t *args)
+{
+  char *value;
+  int ok, ret;
+  uint64_t num_reveals;
+  sr_srv_t *srv = NULL;
+
+  tor_assert(args);
+
+  if (smartlist_len(args) < 2) {
+    goto end;
+  }
+
+  /* First argument is the number of reveal values */
+  num_reveals = tor_parse_uint64(smartlist_get(args, 0),
+                                 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    goto end;
+  }
+  /* Second and last argument is the shared random value it self. */
+  value = smartlist_get(args, 1);
+  if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) {
+    goto end;
+  }
+
+  srv = tor_malloc_zero(sizeof(*srv));
+  srv->num_reveals = num_reveals;
+  /* We subtract one byte from the srclen because the function ignores the
+   * '=' character in the given buffer. This is broken but it's a documented
+   * behavior of the implementation. */
+  ret = base64_decode((char *) srv->value, sizeof(srv->value), value,
+                      SR_SRV_VALUE_BASE64_LEN - 1);
+  if (ret != sizeof(srv->value)) {
+    tor_free(srv);
+    srv = NULL;
+    goto end;
+  }
+ end:
+  return srv;
+}
+
+/** Return the start time of the current SR protocol run. For example, if the
+ *  time is 23/06/2017 23:47:08 and a full SR protocol run is 24 hours, this
+ *  function should return 23/06/2017 00:00:00. */
+time_t
+sr_state_get_start_time_of_current_protocol_run(time_t now)
+{
+  int total_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
+  int voting_interval = get_voting_interval();
+  /* Find the time the current round started. */
+  time_t beginning_of_current_round = get_start_time_of_current_round();
+
+  /* Get current SR protocol round */
+  int current_round = (now / voting_interval) % total_rounds;
+
+  /* Get start time by subtracting the time elapsed from the beginning of the
+     protocol run */
+  time_t time_elapsed_since_start_of_run = current_round * voting_interval;
+  return beginning_of_current_round - time_elapsed_since_start_of_run;
+}
+
+/** Return the time (in seconds) it takes to complete a full SR protocol phase
+ *  (e.g. the commit phase). */
+unsigned int
+sr_state_get_phase_duration(void)
+{
+  return SHARED_RANDOM_N_ROUNDS * get_voting_interval();
+}
+
+/** Return the time (in seconds) it takes to complete a full SR protocol run */
+unsigned int
+sr_state_get_protocol_run_duration(void)
+{
+  int total_protocol_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
+  return total_protocol_rounds * get_voting_interval();
+}
+
diff --git a/src/feature/hs_common/shared_random_client.h b/src/feature/hs_common/shared_random_client.h
new file mode 100644
index 0000000000..079829496c
--- /dev/null
+++ b/src/feature/hs_common/shared_random_client.h
@@ -0,0 +1,47 @@
+/* Copyright (c) 2018-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shared_random_client.h
+ * \brief Header file for shared_random_client.c.
+ **/
+
+#ifndef TOR_SHARED_RANDOM_CLIENT_H
+#define TOR_SHARED_RANDOM_CLIENT_H
+
+/* Dirauth module. */
+#include "or/dirauth/shared_random.h"
+
+/* Helper functions. */
+void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv);
+int get_voting_interval(void);
+
+/* Control port functions. */
+char *sr_get_current_for_control(void);
+char *sr_get_previous_for_control(void);
+
+/* SRV functions. */
+const sr_srv_t *sr_get_current(const networkstatus_t *ns);
+const sr_srv_t *sr_get_previous(const networkstatus_t *ns);
+sr_srv_t *sr_parse_srv(const smartlist_t *args);
+
+/*
+ * Shared Random State API
+ */
+
+/* Each protocol phase has 12 rounds  */
+#define SHARED_RANDOM_N_ROUNDS 12
+/* Number of phase we have in a protocol. */
+#define SHARED_RANDOM_N_PHASES 2
+
+time_t sr_state_get_start_time_of_current_protocol_run(time_t now);
+unsigned int sr_state_get_phase_duration(void);
+unsigned int sr_state_get_protocol_run_duration(void);
+time_t get_start_time_of_current_round(void);
+
+#ifdef TOR_UNIT_TESTS
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* TOR_SHARED_RANDOM_CLIENT_H */
+
diff --git a/src/feature/nodelist/authority_cert_st.h b/src/feature/nodelist/authority_cert_st.h
new file mode 100644
index 0000000000..19c3fda2de
--- /dev/null
+++ b/src/feature/nodelist/authority_cert_st.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef AUTHORITY_CERT_ST_H
+#define AUTHORITY_CERT_ST_H
+
+#include "or/signed_descriptor_st.h"
+
+/** Certificate for v3 directory protocol: binds long-term authority identity
+ * keys to medium-term authority signing keys. */
+struct authority_cert_t {
+  /** Information relating to caching this cert on disk and looking it up. */
+  signed_descriptor_t cache_info;
+  /** This authority's long-term authority identity key. */
+  crypto_pk_t *identity_key;
+  /** This authority's medium-term signing key. */
+  crypto_pk_t *signing_key;
+  /** The digest of <b>signing_key</b> */
+  char signing_key_digest[DIGEST_LEN];
+  /** The listed expiration time of this certificate. */
+  time_t expires;
+  /** This authority's IPv4 address, in host order. */
+  uint32_t addr;
+  /** This authority's directory port. */
+  uint16_t dir_port;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/desc_store_st.h b/src/feature/nodelist/desc_store_st.h
new file mode 100644
index 0000000000..168a83b230
--- /dev/null
+++ b/src/feature/nodelist/desc_store_st.h
@@ -0,0 +1,39 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DESC_STORE_ST_H
+#define DESC_STORE_ST_H
+
+/** Allowable types of desc_store_t. */
+typedef enum store_type_t {
+  ROUTER_STORE = 0,
+  EXTRAINFO_STORE = 1
+} store_type_t;
+
+/** A 'store' is a set of descriptors saved on disk, with accompanying
+ * journal, mmaped as needed, rebuilt as needed. */
+struct desc_store_t {
+  /** Filename (within DataDir) for the store.  We append .tmp to this
+   * filename for a temporary file when rebuilding the store, and .new to this
+   * filename for the journal. */
+  const char *fname_base;
+  /** Human-readable description of what this store contains. */
+  const char *description;
+
+  tor_mmap_t *mmap; /**< A mmap for the main file in the store. */
+
+  store_type_t type; /**< What's stored in this store? */
+
+  /** The size of the router log, in bytes. */
+  size_t journal_len;
+  /** The size of the router store, in bytes. */
+  size_t store_len;
+  /** Total bytes dropped since last rebuild: this is space currently
+   * used in the cache and the journal that could be freed by a rebuild. */
+  size_t bytes_dropped;
+};
+
+#endif
diff --git a/src/feature/nodelist/document_signature_st.h b/src/feature/nodelist/document_signature_st.h
new file mode 100644
index 0000000000..0291e099bf
--- /dev/null
+++ b/src/feature/nodelist/document_signature_st.h
@@ -0,0 +1,29 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DOCUMENT_SIGNATURE_ST_H
+#define DOCUMENT_SIGNATURE_ST_H
+
+/** A signature of some document by an authority. */
+struct document_signature_t {
+  /** Declared SHA-1 digest of this voter's identity key */
+  char identity_digest[DIGEST_LEN];
+  /** Declared SHA-1 digest of signing key used by this voter. */
+  char signing_key_digest[DIGEST_LEN];
+  /** Algorithm used to compute the digest of the document. */
+  digest_algorithm_t alg;
+  /** Signature of the signed thing. */
+  char *signature;
+  /** Length of <b>signature</b> */
+  int signature_len;
+  unsigned int bad_signature : 1; /**< Set to true if we've tried to verify
+                                   * the sig, and we know it's bad. */
+  unsigned int good_signature : 1; /**< Set to true if we've verified the sig
+                                     * as good. */
+};
+
+#endif
+
diff --git a/src/feature/nodelist/extrainfo_st.h b/src/feature/nodelist/extrainfo_st.h
new file mode 100644
index 0000000000..f91bba7b68
--- /dev/null
+++ b/src/feature/nodelist/extrainfo_st.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef EXTRAINFO_ST_H
+#define EXTRAINFO_ST_H
+
+#include "or/signed_descriptor_st.h"
+
+/** Information needed to keep and cache a signed extra-info document. */
+struct extrainfo_t {
+  signed_descriptor_t cache_info;
+  /** SHA256 digest of this document */
+  uint8_t digest256[DIGEST256_LEN];
+  /** The router's nickname. */
+  char nickname[MAX_NICKNAME_LEN+1];
+  /** True iff we found the right key for this extra-info, verified the
+   * signature, and found it to be bad. */
+  unsigned int bad_sig : 1;
+  /** If present, we didn't have the right key to verify this extra-info,
+   * so this is a copy of the signature in the document. */
+  char *pending_sig;
+  /** Length of pending_sig. */
+  size_t pending_sig_len;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c
new file mode 100644
index 0000000000..95c5e8b6f7
--- /dev/null
+++ b/src/feature/nodelist/microdesc.c
@@ -0,0 +1,1059 @@
+/* Copyright (c) 2009-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file microdesc.c
+ *
+ * \brief Implements microdescriptors -- an abbreviated description of
+ *  less-frequently-changing router information.
+ */
+
+#include "or/or.h"
+
+#include "lib/fdio/fdio.h"
+
+#include "or/circuitbuild.h"
+#include "or/config.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/entrynodes.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+
+#include "or/microdesc_st.h"
+#include "or/networkstatus_st.h"
+#include "or/node_st.h"
+#include "or/routerstatus_st.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+/** A data structure to hold a bunch of cached microdescriptors.  There are
+ * two active files in the cache: a "cache file" that we mmap, and a "journal
+ * file" that we append to.  Periodically, we rebuild the cache file to hold
+ * only the microdescriptors that we want to keep */
+struct microdesc_cache_t {
+  /** Map from sha256-digest to microdesc_t for every microdesc_t in the
+   * cache. */
+  HT_HEAD(microdesc_map, microdesc_t) map;
+
+  /** Name of the cache file. */
+  char *cache_fname;
+  /** Name of the journal file. */
+  char *journal_fname;
+  /** Mmap'd contents of the cache file, or NULL if there is none. */
+  tor_mmap_t *cache_content;
+  /** Number of bytes used in the journal file. */
+  size_t journal_len;
+  /** Number of bytes in descriptors removed as too old. */
+  size_t bytes_dropped;
+
+  /** Total bytes of microdescriptor bodies we have added to this cache */
+  uint64_t total_len_seen;
+  /** Total number of microdescriptors we have added to this cache */
+  unsigned n_seen;
+
+  /** True iff we have loaded this cache from disk ever. */
+  int is_loaded;
+};
+
+static microdesc_cache_t *get_microdesc_cache_noload(void);
+
+/** Helper: computes a hash of <b>md</b> to place it in a hash table. */
+static inline unsigned int
+microdesc_hash_(microdesc_t *md)
+{
+  return (unsigned) siphash24g(md->digest, sizeof(md->digest));
+}
+
+/** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table
+ * purposes. */
+static inline int
+microdesc_eq_(microdesc_t *a, microdesc_t *b)
+{
+  return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
+}
+
+HT_PROTOTYPE(microdesc_map, microdesc_t, node,
+             microdesc_hash_, microdesc_eq_)
+HT_GENERATE2(microdesc_map, microdesc_t, node,
+             microdesc_hash_, microdesc_eq_, 0.6,
+             tor_reallocarray_, tor_free_)
+
+/************************* md fetch fail cache *****************************/
+
+/* If we end up with too many outdated dirservers, something probably went
+ * wrong so clean up the list. */
+#define TOO_MANY_OUTDATED_DIRSERVERS 30
+
+/** List of dirservers with outdated microdesc information. The smartlist is
+ *  filled with the hex digests of outdated dirservers. */
+static smartlist_t *outdated_dirserver_list = NULL;
+
+/** Note that we failed to fetch a microdescriptor from the relay with
+ *  <b>relay_digest</b> (of size DIGEST_LEN). */
+void
+microdesc_note_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  /* Don't register outdated dirservers if we don't have a live consensus,
+   * since we might be trying to fetch microdescriptors that are not even
+   * currently active. */
+  if (!networkstatus_get_live_consensus(approx_time())) {
+    return;
+  }
+
+  if (!outdated_dirserver_list) {
+    outdated_dirserver_list = smartlist_new();
+  }
+
+  tor_assert(outdated_dirserver_list);
+
+  /* If the list grows too big, clean it up */
+  if (BUG(smartlist_len(outdated_dirserver_list) >
+          TOO_MANY_OUTDATED_DIRSERVERS)) {
+    microdesc_reset_outdated_dirservers_list();
+  }
+
+  /* Turn the binary relay digest to a hex since smartlists have better support
+   * for strings than digests. */
+  base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Make sure we don't add a dirauth as an outdated dirserver */
+  if (router_get_trusteddirserver_by_digest(relay_digest)) {
+    log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
+    return;
+  }
+
+  /* Don't double-add outdated dirservers */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return;
+  }
+
+  /* Add it to the list of outdated dirservers */
+  smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
+
+  log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
+}
+
+/** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
+ *  outdated dirserver */
+int
+microdesc_relay_is_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  if (!outdated_dirserver_list) {
+    return 0;
+  }
+
+  /* Convert identity digest to hex digest */
+  base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Last time we tried to fetch microdescs, was this directory mirror missing
+   * any mds we asked for? */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/** Reset the list of outdated dirservers. */
+void
+microdesc_reset_outdated_dirservers_list(void)
+{
+  if (!outdated_dirserver_list) {
+    return;
+  }
+
+  SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+  smartlist_clear(outdated_dirserver_list);
+}
+
+/****************************************************************************/
+
+/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
+ * On success, return the total number of bytes written, and set
+ * *<b>annotation_len_out</b> to the number of bytes written as
+ * annotations. */
+static ssize_t
+dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
+{
+  ssize_t r = 0;
+  ssize_t written;
+  if (md->body == NULL) {
+    *annotation_len_out = 0;
+    return 0;
+  }
+  /* XXXX drops unknown annotations. */
+  if (md->last_listed) {
+    char buf[ISO_TIME_LEN+1];
+    char annotation[ISO_TIME_LEN+32];
+    format_iso_time(buf, md->last_listed);
+    tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
+    if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
+      log_warn(LD_DIR,
+               "Couldn't write microdescriptor annotation: %s",
+               strerror(errno));
+      return -1;
+    }
+    r += strlen(annotation);
+    *annotation_len_out = r;
+  } else {
+    *annotation_len_out = 0;
+  }
+
+  md->off = tor_fd_getpos(fd);
+  written = write_all_to_fd(fd, md->body, md->bodylen);
+  if (written != (ssize_t)md->bodylen) {
+    written = written < 0 ? 0 : written;
+    log_warn(LD_DIR,
+             "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
+             (long)written, (unsigned long)md->bodylen,
+             strerror(errno));
+    return -1;
+  }
+  r += md->bodylen;
+  return r;
+}
+
+/** Holds a pointer to the current microdesc_cache_t object, or NULL if no
+ * such object has been allocated. */
+static microdesc_cache_t *the_microdesc_cache = NULL;
+
+/** Return a pointer to the microdescriptor cache, loading it if necessary. */
+microdesc_cache_t *
+get_microdesc_cache(void)
+{
+  microdesc_cache_t *cache = get_microdesc_cache_noload();
+  if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
+    microdesc_cache_reload(cache);
+  }
+  return cache;
+}
+
+/** Return a pointer to the microdescriptor cache, creating (but not loading)
+ * it if necessary. */
+static microdesc_cache_t *
+get_microdesc_cache_noload(void)
+{
+  if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
+    microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
+    HT_INIT(microdesc_map, &cache->map);
+    cache->cache_fname = get_cachedir_fname("cached-microdescs");
+    cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
+    the_microdesc_cache = cache;
+  }
+  return the_microdesc_cache;
+}
+
+/* There are three sources of microdescriptors:
+   1) Generated by us while acting as a directory authority.
+   2) Loaded from the cache on disk.
+   3) Downloaded.
+*/
+
+/** Decode the microdescriptors from the string starting at <b>s</b> and
+ * ending at <b>eos</b>, and store them in <b>cache</b>.  If <b>no_save</b>,
+ * mark them as non-writable to disk.  If <b>where</b> is SAVED_IN_CACHE,
+ * leave their bodies as pointers to the mmap'd cache.  If where is
+ * <b>SAVED_NOWHERE</b>, do not allow annotations.  If listed_at is not -1,
+ * set the last_listed field of every microdesc to listed_at.  If
+ * requested_digests is non-null, then it contains a list of digests we mean
+ * to allow, so we should reject any non-requested microdesc with a different
+ * digest, and alter the list to contain only the digests of those microdescs
+ * we didn't find.
+ * Return a newly allocated list of the added microdescriptors, or NULL  */
+smartlist_t *
+microdescs_add_to_cache(microdesc_cache_t *cache,
+                        const char *s, const char *eos, saved_location_t where,
+                        int no_save, time_t listed_at,
+                        smartlist_t *requested_digests256)
+{
+  void * const DIGEST_REQUESTED = (void*)1;
+  void * const DIGEST_RECEIVED = (void*)2;
+  void * const DIGEST_INVALID = (void*)3;
+
+  smartlist_t *descriptors, *added;
+  const int allow_annotations = (where != SAVED_NOWHERE);
+  smartlist_t *invalid_digests = smartlist_new();
+
+  descriptors = microdescs_parse_from_string(s, eos,
+                                             allow_annotations,
+                                             where, invalid_digests);
+  if (listed_at != (time_t)-1) {
+    SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
+                      md->last_listed = listed_at);
+  }
+  if (requested_digests256) {
+    digest256map_t *requested;
+    requested = digest256map_new();
+    /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
+    SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
+      digest256map_set(requested, cp, DIGEST_REQUESTED));
+    /* Set requested[d] to DIGEST_INVALID for every md we requested which we
+     * will never be able to parse.  Remove the ones we didn't request from
+     * invalid_digests.
+     */
+    SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
+      if (digest256map_get(requested, cp)) {
+        digest256map_set(requested, cp, DIGEST_INVALID);
+      } else {
+        tor_free(cp);
+        SMARTLIST_DEL_CURRENT(invalid_digests, cp);
+      }
+    } SMARTLIST_FOREACH_END(cp);
+    /* Update requested[d] to 2 for the mds we asked for and got. Delete the
+     * ones we never requested from the 'descriptors' smartlist.
+     */
+    SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+      if (digest256map_get(requested, (const uint8_t*)md->digest)) {
+        digest256map_set(requested, (const uint8_t*)md->digest,
+                         DIGEST_RECEIVED);
+      } else {
+        log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
+        microdesc_free(md);
+        SMARTLIST_DEL_CURRENT(descriptors, md);
+      }
+    } SMARTLIST_FOREACH_END(md);
+    /* Remove the ones we got or the invalid ones from requested_digests256.
+     */
+    SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
+      void *status = digest256map_get(requested, cp);
+      if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
+        tor_free(cp);
+        SMARTLIST_DEL_CURRENT(requested_digests256, cp);
+      }
+    } SMARTLIST_FOREACH_END(cp);
+    digest256map_free(requested, NULL);
+  }
+
+  /* For every requested microdescriptor that was unparseable, mark it
+   * as not to be retried. */
+  if (smartlist_len(invalid_digests)) {
+    networkstatus_t *ns =
+      networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
+    if (ns) {
+      SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
+        routerstatus_t *rs =
+          router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
+        if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
+          download_status_mark_impossible(&rs->dl_status);
+        }
+      } SMARTLIST_FOREACH_END(d);
+    }
+  }
+  SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
+  smartlist_free(invalid_digests);
+
+  added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
+  smartlist_free(descriptors);
+  return added;
+}
+
+/** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
+ * a string to decode.  Frees any members of <b>descriptors</b> that it does
+ * not add. */
+smartlist_t *
+microdescs_add_list_to_cache(microdesc_cache_t *cache,
+                             smartlist_t *descriptors, saved_location_t where,
+                             int no_save)
+{
+  smartlist_t *added;
+  open_file_t *open_file = NULL;
+  int fd = -1;
+  //  int n_added = 0;
+  ssize_t size = 0;
+
+  if (where == SAVED_NOWHERE && !no_save) {
+    fd = start_writing_to_file(cache->journal_fname,
+                               OPEN_FLAGS_APPEND|O_BINARY,
+                               0600, &open_file);
+    if (fd < 0) {
+      log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
+               cache->journal_fname, strerror(errno));
+    }
+  }
+
+  added = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+    microdesc_t *md2;
+    md2 = HT_FIND(microdesc_map, &cache->map, md);
+    if (md2) {
+      /* We already had this one. */
+      if (md2->last_listed < md->last_listed)
+        md2->last_listed = md->last_listed;
+      microdesc_free(md);
+      if (where != SAVED_NOWHERE)
+        cache->bytes_dropped += size;
+      continue;
+    }
+
+    /* Okay, it's a new one. */
+    if (fd >= 0) {
+      size_t annotation_len;
+      size = dump_microdescriptor(fd, md, &annotation_len);
+      if (size < 0) {
+        /* we already warned in dump_microdescriptor */
+        abort_writing_to_file(open_file);
+        fd = -1;
+      } else {
+        md->saved_location = SAVED_IN_JOURNAL;
+        cache->journal_len += size;
+      }
+    } else {
+      md->saved_location = where;
+    }
+
+    md->no_save = no_save;
+
+    HT_INSERT(microdesc_map, &cache->map, md);
+    md->held_in_map = 1;
+    smartlist_add(added, md);
+    ++cache->n_seen;
+    cache->total_len_seen += md->bodylen;
+  } SMARTLIST_FOREACH_END(md);
+
+  if (fd >= 0) {
+    if (finish_writing_to_file(open_file) < 0) {
+      log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
+               strerror(errno));
+      smartlist_clear(added);
+      return added;
+    }
+  }
+
+  {
+    networkstatus_t *ns = networkstatus_get_latest_consensus();
+    if (ns && ns->flavor == FLAV_MICRODESC)
+      SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md));
+  }
+
+  if (smartlist_len(added))
+    router_dir_info_changed();
+
+  return added;
+}
+
+/** Remove every microdescriptor in <b>cache</b>. */
+void
+microdesc_cache_clear(microdesc_cache_t *cache)
+{
+  microdesc_t **entry, **next;
+
+  for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
+    microdesc_t *md = *entry;
+    next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
+    md->held_in_map = 0;
+    microdesc_free(md);
+  }
+  HT_CLEAR(microdesc_map, &cache->map);
+  if (cache->cache_content) {
+    int res = tor_munmap_file(cache->cache_content);
+    if (res != 0) {
+      log_warn(LD_FS,
+               "tor_munmap_file() failed clearing microdesc cache; "
+               "we are probably about to leak memory.");
+      /* TODO something smarter? */
+    }
+    cache->cache_content = NULL;
+  }
+  cache->total_len_seen = 0;
+  cache->n_seen = 0;
+  cache->bytes_dropped = 0;
+}
+
+/** Reload the contents of <b>cache</b> from disk.  If it is empty, load it
+ * for the first time.  Return 0 on success, -1 on failure. */
+int
+microdesc_cache_reload(microdesc_cache_t *cache)
+{
+  struct stat st;
+  char *journal_content;
+  smartlist_t *added;
+  tor_mmap_t *mm;
+  int total = 0;
+
+  microdesc_cache_clear(cache);
+
+  cache->is_loaded = 1;
+
+  mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
+  if (mm) {
+    added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
+                                    SAVED_IN_CACHE, 0, -1, NULL);
+    if (added) {
+      total += smartlist_len(added);
+      smartlist_free(added);
+    }
+  }
+
+  journal_content = read_file_to_str(cache->journal_fname,
+                                     RFTS_IGNORE_MISSING, &st);
+  if (journal_content) {
+    cache->journal_len = (size_t) st.st_size;
+    added = microdescs_add_to_cache(cache, journal_content,
+                                    journal_content+st.st_size,
+                                    SAVED_IN_JOURNAL, 0, -1, NULL);
+    if (added) {
+      total += smartlist_len(added);
+      smartlist_free(added);
+    }
+    tor_free(journal_content);
+  }
+  log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
+           total);
+
+  microdesc_cache_rebuild(cache, 0 /* don't force */);
+
+  return 0;
+}
+
+/** By default, we remove any microdescriptors that have gone at least this
+ * long without appearing in a current consensus. */
+#define TOLERATE_MICRODESC_AGE (7*24*60*60)
+
+/** Remove all microdescriptors from <b>cache</b> that haven't been listed for
+ * a long time.  Does not rebuild the cache on disk.  If <b>cutoff</b> is
+ * positive, specifically remove microdescriptors that have been unlisted
+ * since <b>cutoff</b>.  If <b>force</b> is true, remove microdescriptors even
+ * if we have no current live microdescriptor consensus.
+ */
+void
+microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
+{
+  microdesc_t **mdp, *victim;
+  int dropped=0, kept=0;
+  size_t bytes_dropped = 0;
+  time_t now = time(NULL);
+
+  /* If we don't know a live consensus, don't believe last_listed values: we
+   * might be starting up after being down for a while. */
+  if (! force &&
+      ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
+      return;
+
+  if (cutoff <= 0)
+    cutoff = now - TOLERATE_MICRODESC_AGE;
+
+  for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
+    const int is_old = (*mdp)->last_listed < cutoff;
+    const unsigned held_by_nodes = (*mdp)->held_by_nodes;
+    if (is_old && !held_by_nodes) {
+      ++dropped;
+      victim = *mdp;
+      mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
+      victim->held_in_map = 0;
+      bytes_dropped += victim->bodylen;
+      microdesc_free(victim);
+    } else {
+      if (is_old) {
+        /* It's old, but it has held_by_nodes set.  That's not okay. */
+        /* Let's try to diagnose and fix #7164 . */
+        smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp);
+        const networkstatus_t *ns = networkstatus_get_latest_consensus();
+        long networkstatus_age = -1;
+        const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
+        if (ns) {
+          networkstatus_age = now - ns->valid_after;
+        }
+        log_warn(LD_BUG, "Microdescriptor seemed very old "
+                 "(last listed %d hours ago vs %d hour cutoff), but is still "
+                 "marked as being held by %d node(s). I found %d node(s) "
+                 "holding it. Current networkstatus is %ld hours old. "
+                 "Hashtable badness is %d.",
+                 (int)((now - (*mdp)->last_listed) / 3600),
+                 (int)((now - cutoff) / 3600),
+                 held_by_nodes,
+                 smartlist_len(nodes),
+                 networkstatus_age / 3600,
+                 ht_badness);
+
+        SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+          const char *rs_match = "No RS";
+          const char *rs_present = "";
+          if (node->rs) {
+            if (tor_memeq(node->rs->descriptor_digest,
+                          (*mdp)->digest, DIGEST256_LEN)) {
+              rs_match = "Microdesc digest in RS matches";
+            } else {
+              rs_match = "Microdesc digest in RS does match";
+            }
+            if (ns) {
+              /* This should be impossible, but let's see! */
+              rs_present = " RS not present in networkstatus.";
+              SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, {
+                if (rs == node->rs) {
+                  rs_present = " RS okay in networkstatus.";
+                }
+              });
+            }
+          }
+          log_warn(LD_BUG, "  [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
+                   node_sl_idx,
+                   hex_str(node->identity, DIGEST_LEN),
+                   node->md, node->rs, node->ri, rs_match, rs_present);
+        } SMARTLIST_FOREACH_END(node);
+        smartlist_free(nodes);
+        (*mdp)->last_listed = now;
+      }
+
+      ++kept;
+      mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
+    }
+  }
+
+  if (dropped) {
+    log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
+             dropped,dropped+kept);
+    cache->bytes_dropped += bytes_dropped;
+  }
+}
+
+static int
+should_rebuild_md_cache(microdesc_cache_t *cache)
+{
+    const size_t old_len =
+      cache->cache_content ? cache->cache_content->size : 0;
+    const size_t journal_len = cache->journal_len;
+    const size_t dropped = cache->bytes_dropped;
+
+    if (journal_len < 16384)
+      return 0; /* Don't bother, not enough has happened yet. */
+    if (dropped > (journal_len + old_len) / 3)
+      return 1; /* We could save 1/3 or more of the currently used space. */
+    if (journal_len > old_len / 2)
+      return 1; /* We should append to the regular file */
+
+    return 0;
+}
+
+/**
+ * Mark <b>md</b> as having no body, and release any storage previously held
+ * by its body.
+ */
+static void
+microdesc_wipe_body(microdesc_t *md)
+{
+  if (!md)
+    return;
+
+  if (md->saved_location != SAVED_IN_CACHE)
+    tor_free(md->body);
+
+  md->off = 0;
+  md->saved_location = SAVED_NOWHERE;
+  md->body = NULL;
+  md->bodylen = 0;
+  md->no_save = 1;
+}
+
+/** Regenerate the main cache file for <b>cache</b>, clear the journal file,
+ * and update every microdesc_t in the cache with pointers to its new
+ * location.  If <b>force</b> is true, do this unconditionally.  If
+ * <b>force</b> is false, do it only if we expect to save space on disk. */
+int
+microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
+{
+  open_file_t *open_file;
+  int fd = -1, res;
+  microdesc_t **mdp;
+  smartlist_t *wrote;
+  ssize_t size;
+  off_t off = 0, off_real;
+  int orig_size, new_size;
+
+  if (cache == NULL) {
+    cache = the_microdesc_cache;
+    if (cache == NULL)
+      return 0;
+  }
+
+  /* Remove dead descriptors */
+  microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
+
+  if (!force && !should_rebuild_md_cache(cache))
+    return 0;
+
+  log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
+
+  orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
+  orig_size += (int)cache->journal_len;
+
+  fd = start_writing_to_file(cache->cache_fname,
+                             OPEN_FLAGS_REPLACE|O_BINARY,
+                             0600, &open_file);
+  if (fd < 0)
+    return -1;
+
+  wrote = smartlist_new();
+
+  HT_FOREACH(mdp, microdesc_map, &cache->map) {
+    microdesc_t *md = *mdp;
+    size_t annotation_len;
+    if (md->no_save || !md->body)
+      continue;
+
+    size = dump_microdescriptor(fd, md, &annotation_len);
+    if (size < 0) {
+      microdesc_wipe_body(md);
+
+      /* rewind, in case it was a partial write. */
+      tor_fd_setpos(fd, off);
+      continue;
+    }
+    tor_assert(((size_t)size) == annotation_len + md->bodylen);
+    md->off = off + annotation_len;
+    off += size;
+    off_real = tor_fd_getpos(fd);
+    if (off_real != off) {
+      log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
+               "By my count, I'm at %"PRId64
+               ", but I should be at %"PRId64,
+               (off), (off_real));
+      if (off_real >= 0)
+        off = off_real;
+    }
+    if (md->saved_location != SAVED_IN_CACHE) {
+      tor_free(md->body);
+      md->saved_location = SAVED_IN_CACHE;
+    }
+    smartlist_add(wrote, md);
+  }
+
+  /* We must do this unmap _before_ we call finish_writing_to_file(), or
+   * windows will not actually replace the file. */
+  if (cache->cache_content) {
+    res = tor_munmap_file(cache->cache_content);
+    if (res != 0) {
+      log_warn(LD_FS,
+               "Failed to unmap old microdescriptor cache while rebuilding");
+    }
+    cache->cache_content = NULL;
+  }
+
+  if (finish_writing_to_file(open_file) < 0) {
+    log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
+             strerror(errno));
+    /* Okay. Let's prevent from making things worse elsewhere. */
+    cache->cache_content = NULL;
+    HT_FOREACH(mdp, microdesc_map, &cache->map) {
+      microdesc_t *md = *mdp;
+      if (md->saved_location == SAVED_IN_CACHE) {
+        microdesc_wipe_body(md);
+      }
+    }
+    smartlist_free(wrote);
+    return -1;
+  }
+
+  cache->cache_content = tor_mmap_file(cache->cache_fname);
+
+  if (!cache->cache_content && smartlist_len(wrote)) {
+    log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
+            cache->cache_fname);
+    smartlist_free(wrote);
+    return -1;
+  }
+  SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
+    tor_assert(md->saved_location == SAVED_IN_CACHE);
+    md->body = (char*)cache->cache_content->data + md->off;
+    if (PREDICT_UNLIKELY(
+             md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
+      /* XXXX once bug 2022 is solved, we can kill this block and turn it
+       * into just the tor_assert(fast_memeq) */
+      off_t avail = cache->cache_content->size - md->off;
+      char *bad_str;
+      tor_assert(avail >= 0);
+      bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
+      log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
+              " At offset %d, I expected to find a microdescriptor starting "
+              " with \"onion-key\".  Instead I got %s.",
+              (int)md->off, escaped(bad_str));
+      tor_free(bad_str);
+      tor_assert(fast_memeq(md->body, "onion-key", 9));
+    }
+  } SMARTLIST_FOREACH_END(md);
+
+  smartlist_free(wrote);
+
+  write_str_to_file(cache->journal_fname, "", 1);
+  cache->journal_len = 0;
+  cache->bytes_dropped = 0;
+
+  new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
+  log_info(LD_DIR, "Done rebuilding microdesc cache. "
+           "Saved %d bytes; %d still used.",
+           orig_size-new_size, new_size);
+
+  return 0;
+}
+
+/** Make sure that the reference count of every microdescriptor in cache is
+ * accurate. */
+void
+microdesc_check_counts(void)
+{
+  microdesc_t **mdp;
+  if (!the_microdesc_cache)
+    return;
+
+  HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
+    microdesc_t *md = *mdp;
+    unsigned int found=0;
+    const smartlist_t *nodes = nodelist_get_list();
+    SMARTLIST_FOREACH(nodes, node_t *, node, {
+        if (node->md == md) {
+          ++found;
+        }
+      });
+    tor_assert(found == md->held_by_nodes);
+  }
+}
+
+/** Deallocate a single microdescriptor.  Note: the microdescriptor MUST have
+ * previously been removed from the cache if it had ever been inserted. */
+void
+microdesc_free_(microdesc_t *md, const char *fname, int lineno)
+{
+  if (!md)
+    return;
+
+  /* Make sure that the microdesc was really removed from the appropriate data
+     structures. */
+  if (md->held_in_map) {
+    microdesc_cache_t *cache = get_microdesc_cache_noload();
+    microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
+    if (md2 == md) {
+      log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
+               "in microdesc_map", fname, lineno);
+      HT_REMOVE(microdesc_map, &cache->map, md);
+    } else {
+      log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
+               "set, but microdesc was not in the map.", fname, lineno);
+    }
+    tor_fragile_assert();
+  }
+  if (md->held_by_nodes) {
+    microdesc_cache_t *cache = get_microdesc_cache_noload();
+    int found=0;
+    const smartlist_t *nodes = nodelist_get_list();
+    const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
+    SMARTLIST_FOREACH(nodes, node_t *, node, {
+        if (node->md == md) {
+          ++found;
+          node->md = NULL;
+        }
+      });
+    if (found) {
+      log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
+               "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
+               fname, lineno, found, md->held_by_nodes, ht_badness);
+    } else {
+      log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
+               "set to %u, but md was not referenced by any nodes. "
+               "ht_badness == %d",
+               fname, lineno, md->held_by_nodes, ht_badness);
+    }
+    tor_fragile_assert();
+  }
+  //tor_assert(md->held_in_map == 0);
+  //tor_assert(md->held_by_nodes == 0);
+
+  if (md->onion_pkey)
+    crypto_pk_free(md->onion_pkey);
+  tor_free(md->onion_curve25519_pkey);
+  tor_free(md->ed25519_identity_pkey);
+  if (md->body && md->saved_location != SAVED_IN_CACHE)
+    tor_free(md->body);
+
+  if (md->family) {
+    SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
+    smartlist_free(md->family);
+  }
+  short_policy_free(md->exit_policy);
+  short_policy_free(md->ipv6_exit_policy);
+
+  tor_free(md);
+}
+
+/** Free all storage held in the microdesc.c module. */
+void
+microdesc_free_all(void)
+{
+  if (the_microdesc_cache) {
+    microdesc_cache_clear(the_microdesc_cache);
+    tor_free(the_microdesc_cache->cache_fname);
+    tor_free(the_microdesc_cache->journal_fname);
+    tor_free(the_microdesc_cache);
+  }
+
+  if (outdated_dirserver_list) {
+    SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+    smartlist_free(outdated_dirserver_list);
+  }
+}
+
+/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
+ * <b>d</b>, return it.  Otherwise return NULL. */
+microdesc_t *
+microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
+{
+  microdesc_t *md, search;
+  if (!cache)
+    cache = get_microdesc_cache();
+  memcpy(search.digest, d, DIGEST256_LEN);
+  md = HT_FIND(microdesc_map, &cache->map, &search);
+  return md;
+}
+
+/** Return a smartlist of all the sha256 digest of the microdescriptors that
+ * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
+ * to internals of <b>ns</b>; you should not free the members of the resulting
+ * smartlist.  Omit all microdescriptors whose digest appear in <b>skip</b>. */
+smartlist_t *
+microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
+                                 int downloadable_only, digest256map_t *skip)
+{
+  smartlist_t *result = smartlist_new();
+  time_t now = time(NULL);
+  tor_assert(ns->flavor == FLAV_MICRODESC);
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
+      continue;
+    if (downloadable_only &&
+        !download_status_is_ready(&rs->dl_status, now))
+      continue;
+    if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
+      continue;
+    if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
+      continue;
+    /* XXXX Also skip if we're a noncache and wouldn't use this router.
+     * XXXX NM Microdesc
+     */
+    smartlist_add(result, rs->descriptor_digest);
+  } SMARTLIST_FOREACH_END(rs);
+  return result;
+}
+
+/** Launch download requests for microdescriptors as appropriate.
+ *
+ * Specifically, we should launch download requests if we are configured to
+ * download mirodescriptors, and there are some microdescriptors listed in the
+ * current microdesc consensus that we don't have, and either we never asked
+ * for them, or we failed to download them but we're willing to retry.
+ */
+void
+update_microdesc_downloads(time_t now)
+{
+  const or_options_t *options = get_options();
+  networkstatus_t *consensus;
+  smartlist_t *missing;
+  digest256map_t *pending;
+
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+  if (directory_too_idle_to_fetch_descriptors(options, now))
+    return;
+
+  consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
+  if (!consensus)
+    return;
+
+  if (!we_fetch_microdescriptors(options))
+    return;
+
+  pending = digest256map_new();
+  list_pending_microdesc_downloads(pending);
+
+  missing = microdesc_list_missing_digest256(consensus,
+                                             get_microdesc_cache(),
+                                             1,
+                                             pending);
+  digest256map_free(pending, NULL);
+
+  launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
+                              missing, NULL, now);
+
+  smartlist_free(missing);
+}
+
+/** For every microdescriptor listed in the current microdescriptor consensus,
+ * update its last_listed field to be at least as recent as the publication
+ * time of the current microdescriptor consensus.
+ */
+void
+update_microdescs_from_networkstatus(time_t now)
+{
+  microdesc_cache_t *cache = get_microdesc_cache();
+  microdesc_t *md;
+  networkstatus_t *ns =
+    networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
+
+  if (! ns)
+    return;
+
+  tor_assert(ns->flavor == FLAV_MICRODESC);
+
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
+    if (md && ns->valid_after > md->last_listed)
+      md->last_listed = ns->valid_after;
+  } SMARTLIST_FOREACH_END(rs);
+}
+
+/** Return true iff we should prefer to use microdescriptors rather than
+ * routerdescs for building circuits. */
+int
+we_use_microdescriptors_for_circuits(const or_options_t *options)
+{
+  if (options->UseMicrodescriptors == 0)
+    return 0; /* the user explicitly picked no */
+  return 1; /* yes and auto both mean yes */
+}
+
+/** Return true iff we should try to download microdescriptors at all. */
+int
+we_fetch_microdescriptors(const or_options_t *options)
+{
+  if (directory_caches_dir_info(options))
+    return 1;
+  if (options->FetchUselessDescriptors)
+    return 1;
+  return we_use_microdescriptors_for_circuits(options);
+}
+
+/** Return true iff we should try to download router descriptors at all. */
+int
+we_fetch_router_descriptors(const or_options_t *options)
+{
+  if (directory_caches_dir_info(options))
+    return 1;
+  if (options->FetchUselessDescriptors)
+    return 1;
+  return ! we_use_microdescriptors_for_circuits(options);
+}
+
+/** Return the consensus flavor we actually want to use to build circuits. */
+MOCK_IMPL(int,
+usable_consensus_flavor,(void))
+{
+  if (we_use_microdescriptors_for_circuits(get_options())) {
+    return FLAV_MICRODESC;
+  } else {
+    return FLAV_NS;
+  }
+}
diff --git a/src/feature/nodelist/microdesc.h b/src/feature/nodelist/microdesc.h
new file mode 100644
index 0000000000..f11b841cf1
--- /dev/null
+++ b/src/feature/nodelist/microdesc.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file microdesc.h
+ * \brief Header file for microdesc.c.
+ **/
+
+#ifndef TOR_MICRODESC_H
+#define TOR_MICRODESC_H
+
+microdesc_cache_t *get_microdesc_cache(void);
+
+void microdesc_check_counts(void);
+
+smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache,
+                        const char *s, const char *eos, saved_location_t where,
+                        int no_save, time_t listed_at,
+                        smartlist_t *requested_digests256);
+smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache,
+                        smartlist_t *descriptors, saved_location_t where,
+                        int no_save);
+
+void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force);
+int microdesc_cache_rebuild(microdesc_cache_t *cache, int force);
+int microdesc_cache_reload(microdesc_cache_t *cache);
+void microdesc_cache_clear(microdesc_cache_t *cache);
+
+microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache,
+                                                 const char *d);
+
+smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns,
+                                              microdesc_cache_t *cache,
+                                              int downloadable_only,
+                                              digest256map_t *skip);
+
+void microdesc_free_(microdesc_t *md, const char *fname, int line);
+#define microdesc_free(md) do {                 \
+    microdesc_free_((md), __FILE__, __LINE__);  \
+    (md) = NULL;                                \
+  } while (0)
+void microdesc_free_all(void);
+
+void update_microdesc_downloads(time_t now);
+void update_microdescs_from_networkstatus(time_t now);
+
+MOCK_DECL(int, usable_consensus_flavor,(void));
+int we_fetch_microdescriptors(const or_options_t *options);
+int we_fetch_router_descriptors(const or_options_t *options);
+int we_use_microdescriptors_for_circuits(const or_options_t *options);
+
+void microdesc_note_outdated_dirserver(const char *relay_digest);
+int microdesc_relay_is_outdated_dirserver(const char *relay_digest);
+void microdesc_reset_outdated_dirservers_list(void);
+
+#endif /* !defined(TOR_MICRODESC_H) */
+
diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h
new file mode 100644
index 0000000000..e9dc3e0174
--- /dev/null
+++ b/src/feature/nodelist/microdesc_st.h
@@ -0,0 +1,74 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef MICRODESC_ST_H
+#define MICRODESC_ST_H
+
+struct curve25519_public_key_t;
+struct ed25519_public_key_t;
+struct short_policy_t;
+
+/** A microdescriptor is the smallest amount of information needed to build a
+ * circuit through a router.  They are generated by the directory authorities,
+ * using information from the uploaded routerinfo documents.  They are not
+ * self-signed, but are rather authenticated by having their hash in a signed
+ * networkstatus document. */
+struct microdesc_t {
+  /** Hashtable node, used to look up the microdesc by its digest. */
+  HT_ENTRY(microdesc_t) node;
+
+  /* Cache information */
+
+  /**  When was this microdescriptor last listed in a consensus document?
+   * Once a microdesc has been unlisted long enough, we can drop it.
+   */
+  time_t last_listed;
+  /** Where is this microdescriptor currently stored? */
+  saved_location_bitfield_t saved_location : 3;
+  /** If true, do not attempt to cache this microdescriptor on disk. */
+  unsigned int no_save : 1;
+  /** If true, this microdesc has an entry in the microdesc_map */
+  unsigned int held_in_map : 1;
+  /** Reference count: how many node_ts have a reference to this microdesc? */
+  unsigned int held_by_nodes;
+
+  /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the
+   * microdescriptor in the cache. */
+  off_t off;
+
+  /* The string containing the microdesc. */
+
+  /** A pointer to the encoded body of the microdescriptor.  If the
+   * saved_location is SAVED_IN_CACHE, then the body is a pointer into an
+   * mmap'd region.  Otherwise, it is a malloc'd string.  The string might not
+   * be NUL-terminated; take the length from <b>bodylen</b>. */
+  char *body;
+  /** The length of the microdescriptor in <b>body</b>. */
+  size_t bodylen;
+  /** A SHA256-digest of the microdescriptor. */
+  char digest[DIGEST256_LEN];
+
+  /* Fields in the microdescriptor. */
+
+  /** As routerinfo_t.onion_pkey */
+  crypto_pk_t *onion_pkey;
+  /** As routerinfo_t.onion_curve25519_pkey */
+  struct curve25519_public_key_t *onion_curve25519_pkey;
+  /** Ed25519 identity key, if included. */
+  struct ed25519_public_key_t *ed25519_identity_pkey;
+  /** As routerinfo_t.ipv6_addr */
+  tor_addr_t ipv6_addr;
+  /** As routerinfo_t.ipv6_orport */
+  uint16_t ipv6_orport;
+  /** As routerinfo_t.family */
+  smartlist_t *family;
+  /** IPv4 exit policy summary */
+  struct short_policy_t *exit_policy;
+  /** IPv6 exit policy summary */
+  struct short_policy_t *ipv6_exit_policy;
+};
+
+#endif
diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c
new file mode 100644
index 0000000000..f91e46cdd7
--- /dev/null
+++ b/src/feature/nodelist/networkstatus.c
@@ -0,0 +1,2726 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file networkstatus.c
+ * \brief Functions and structures for handling networkstatus documents as a
+ * client or as a directory cache.
+ *
+ * A consensus networkstatus object is created by the directory
+ * authorities.  It authenticates a set of network parameters--most
+ * importantly, the list of all the relays in the network.  This list
+ * of relays is represented as an array of routerstatus_t objects.
+ *
+ * There are currently two flavors of consensus.  With the older "NS"
+ * flavor, each relay is associated with a digest of its router
+ * descriptor. Tor instances that use this consensus keep the list of
+ * router descriptors as routerinfo_t objects stored and managed in
+ * routerlist.c.  With the newer "microdesc" flavor, each relay is
+ * associated with a digest of the microdescriptor that the authorities
+ * made for it.  These are stored and managed in microdesc.c. Information
+ * about the router is divided between the the networkstatus and the
+ * microdescriptor according to the general rule that microdescriptors
+ * should hold information that changes much less frequently than the
+ * information in the networkstatus.
+ *
+ * Modern clients use microdescriptor networkstatuses.  Directory caches
+ * need to keep both kinds of networkstatus document, so they can serve them.
+ *
+ * This module manages fetching, holding, storing, updating, and
+ * validating networkstatus objects.  The download-and-validate process
+ * is slightly complicated by the fact that the keys you need to
+ * validate a consensus are stored in the authority certificates, which
+ * you might not have yet when you download the consensus.
+ */
+
+#define NETWORKSTATUS_PRIVATE
+#include "or/or.h"
+#include "or/bridges.h"
+#include "or/channel.h"
+#include "or/circuitmux.h"
+#include "or/circuitmux_ewma.h"
+#include "or/circuitstats.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/connection_edge.h"
+#include "or/connection_or.h"
+#include "or/consdiffmgr.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/dos.h"
+#include "or/entrynodes.h"
+#include "or/hibernate.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/protover.h"
+#include "or/relay.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/scheduler.h"
+#include "or/transports.h"
+#include "or/torcert.h"
+#include "or/channelpadding.h"
+#include "or/voting_schedule.h"
+
+#include "or/dirauth/dirvote.h"
+#include "or/dirauth/mode.h"
+#include "or/dirauth/shared_random.h"
+
+#include "or/authority_cert_st.h"
+#include "or/dir_connection_st.h"
+#include "or/dir_server_st.h"
+#include "or/document_signature_st.h"
+#include "or/networkstatus_st.h"
+#include "or/networkstatus_voter_info_st.h"
+#include "or/ns_detached_signatures_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/vote_microdesc_hash_st.h"
+#include "or/vote_routerstatus_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/** Most recently received and validated v3 "ns"-flavored consensus network
+ * status. */
+STATIC networkstatus_t *current_ns_consensus = NULL;
+
+/** Most recently received and validated v3 "microdesc"-flavored consensus
+ * network status. */
+STATIC networkstatus_t *current_md_consensus = NULL;
+
+/** A v3 consensus networkstatus that we've received, but which we don't
+ * have enough certificates to be happy about. */
+typedef struct consensus_waiting_for_certs_t {
+  /** The consensus itself. */
+  networkstatus_t *consensus;
+  /** The encoded version of the consensus, nul-terminated. */
+  char *body;
+  /** When did we set the current value of consensus_waiting_for_certs?  If
+   * this is too recent, we shouldn't try to fetch a new consensus for a
+   * little while, to give ourselves time to get certificates for this one. */
+  time_t set_at;
+  /** Set to 1 if we've been holding on to it for so long we should maybe
+   * treat it as being bad. */
+  int dl_failed;
+} consensus_waiting_for_certs_t;
+
+/** An array, for each flavor of consensus we might want, of consensuses that
+ * we have downloaded, but which we cannot verify due to having insufficient
+ * authority certificates. */
+static consensus_waiting_for_certs_t
+       consensus_waiting_for_certs[N_CONSENSUS_FLAVORS];
+
+/** A time before which we shouldn't try to replace the current consensus:
+ * this will be at some point after the next consensus becomes valid, but
+ * before the current consensus becomes invalid. */
+static time_t time_to_download_next_consensus[N_CONSENSUS_FLAVORS];
+/** Download status for the current consensus networkstatus. */
+static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS] =
+  {
+    { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
+      DL_SCHED_INCREMENT_FAILURE, 0, 0 },
+    { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
+      DL_SCHED_INCREMENT_FAILURE, 0, 0 },
+  };
+
+#define N_CONSENSUS_BOOTSTRAP_SCHEDULES 2
+#define CONSENSUS_BOOTSTRAP_SOURCE_AUTHORITY 0
+#define CONSENSUS_BOOTSTRAP_SOURCE_ANY_DIRSERVER  1
+
+/* Using DL_SCHED_INCREMENT_ATTEMPT on these schedules means that
+ * download_status_increment_failure won't increment these entries.
+ * However, any bootstrap connection failures that occur after we have
+ * a valid consensus will count against the failure counts on the non-bootstrap
+ * schedules. There should only be one of these, as all the others will have
+ * been cancelled. (This doesn't seem to be a significant issue.) */
+static download_status_t
+              consensus_bootstrap_dl_status[N_CONSENSUS_BOOTSTRAP_SCHEDULES] =
+  {
+    { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY,
+      DL_SCHED_INCREMENT_ATTEMPT, 0, 0 },
+    /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */
+    { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
+      DL_SCHED_INCREMENT_ATTEMPT, 0, 0 },
+  };
+
+/** True iff we have logged a warning about this OR's version being older than
+ * listed by the authorities. */
+static int have_warned_about_old_version = 0;
+/** True iff we have logged a warning about this OR's version being newer than
+ * listed by the authorities. */
+static int have_warned_about_new_version = 0;
+
+static void update_consensus_bootstrap_multiple_downloads(
+                                                  time_t now,
+                                                  const or_options_t *options);
+static int networkstatus_check_required_protocols(const networkstatus_t *ns,
+                                                  int client_mode,
+                                                  char **warning_out);
+
+/** Forget that we've warned about anything networkstatus-related, so we will
+ * give fresh warnings if the same behavior happens again. */
+void
+networkstatus_reset_warnings(void)
+{
+  SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node,
+                    node->name_lookup_warned = 0);
+
+  have_warned_about_old_version = 0;
+  have_warned_about_new_version = 0;
+}
+
+/** Reset the descriptor download failure count on all networkstatus docs, so
+ * that we can retry any long-failed documents immediately.
+ */
+void
+networkstatus_reset_download_failures(void)
+{
+  int i;
+
+  log_debug(LD_GENERAL,
+            "In networkstatus_reset_download_failures()");
+
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i)
+    download_status_reset(&consensus_dl_status[i]);
+
+  for (i=0; i < N_CONSENSUS_BOOTSTRAP_SCHEDULES; ++i)
+    download_status_reset(&consensus_bootstrap_dl_status[i]);
+}
+
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static char *
+networkstatus_read_cached_consensus_impl(int flav,
+                                         const char *flavorname,
+                                         int unverified_consensus)
+{
+  char buf[128];
+  const char *prefix;
+  if (unverified_consensus) {
+    prefix = "unverified";
+  } else {
+    prefix = "cached";
+  }
+  if (flav == FLAV_NS) {
+    tor_snprintf(buf, sizeof(buf), "%s-consensus", prefix);
+  } else {
+    tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
+  }
+
+  char *filename = get_cachedir_fname(buf);
+  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  tor_free(filename);
+  return result;
+}
+
+/** Return a new string containing the current cached consensus of flavor
+ * <b>flavorname</b>. */
+char *
+networkstatus_read_cached_consensus(const char *flavorname)
+ {
+  int flav = networkstatus_parse_flavor_name(flavorname);
+  if (flav < 0)
+    return NULL;
+  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+}
+
+/** Read every cached v3 consensus networkstatus from the disk. */
+int
+router_reload_consensus_networkstatus(void)
+{
+  const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS;
+  int flav;
+
+  /* FFFF Suppress warnings if cached consensus is bad? */
+  for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavor = networkstatus_get_flavor_name(flav);
+    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
+    if (s) {
+      if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
+        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
+                 flavor);
+      }
+      tor_free(s);
+    }
+
+    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
+    if (s) {
+      if (networkstatus_set_current_consensus(s, flavor,
+                                     flags | NSSET_WAS_WAITING_FOR_CERTS,
+                                     NULL)) {
+        log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
+                 "from cache", flavor);
+      }
+      tor_free(s);
+    }
+  }
+
+  update_certificate_downloads(time(NULL));
+
+  routers_update_all_from_networkstatus(time(NULL), 3);
+  update_microdescs_from_networkstatus(time(NULL));
+
+  return 0;
+}
+
+/** Free all storage held by the vote_routerstatus object <b>rs</b>. */
+void
+vote_routerstatus_free_(vote_routerstatus_t *rs)
+{
+  vote_microdesc_hash_t *h, *next;
+  if (!rs)
+    return;
+  tor_free(rs->version);
+  tor_free(rs->protocols);
+  tor_free(rs->status.exitsummary);
+  for (h = rs->microdesc; h; h = next) {
+    tor_free(h->microdesc_hash_line);
+    next = h->next;
+    tor_free(h);
+  }
+  tor_free(rs);
+}
+
+/** Free all storage held by the routerstatus object <b>rs</b>. */
+void
+routerstatus_free_(routerstatus_t *rs)
+{
+  if (!rs)
+    return;
+  tor_free(rs->exitsummary);
+  tor_free(rs);
+}
+
+/** Free all storage held in <b>sig</b> */
+void
+document_signature_free_(document_signature_t *sig)
+{
+  tor_free(sig->signature);
+  tor_free(sig);
+}
+
+/** Return a newly allocated copy of <b>sig</b> */
+document_signature_t *
+document_signature_dup(const document_signature_t *sig)
+{
+  document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t));
+  if (r->signature)
+    r->signature = tor_memdup(sig->signature, sig->signature_len);
+  return r;
+}
+
+/** Free all storage held in <b>ns</b>. */
+void
+networkstatus_vote_free_(networkstatus_t *ns)
+{
+  if (!ns)
+    return;
+
+  tor_free(ns->client_versions);
+  tor_free(ns->server_versions);
+  tor_free(ns->recommended_client_protocols);
+  tor_free(ns->recommended_relay_protocols);
+  tor_free(ns->required_client_protocols);
+  tor_free(ns->required_relay_protocols);
+
+  if (ns->known_flags) {
+    SMARTLIST_FOREACH(ns->known_flags, char *, c, tor_free(c));
+    smartlist_free(ns->known_flags);
+  }
+  if (ns->weight_params) {
+    SMARTLIST_FOREACH(ns->weight_params, char *, c, tor_free(c));
+    smartlist_free(ns->weight_params);
+  }
+  if (ns->net_params) {
+    SMARTLIST_FOREACH(ns->net_params, char *, c, tor_free(c));
+    smartlist_free(ns->net_params);
+  }
+  if (ns->supported_methods) {
+    SMARTLIST_FOREACH(ns->supported_methods, char *, c, tor_free(c));
+    smartlist_free(ns->supported_methods);
+  }
+  if (ns->package_lines) {
+    SMARTLIST_FOREACH(ns->package_lines, char *, c, tor_free(c));
+    smartlist_free(ns->package_lines);
+  }
+  if (ns->voters) {
+    SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) {
+      tor_free(voter->nickname);
+      tor_free(voter->address);
+      tor_free(voter->contact);
+      if (voter->sigs) {
+        SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                          document_signature_free(sig));
+        smartlist_free(voter->sigs);
+      }
+      tor_free(voter);
+    } SMARTLIST_FOREACH_END(voter);
+    smartlist_free(ns->voters);
+  }
+  authority_cert_free(ns->cert);
+
+  if (ns->routerstatus_list) {
+    if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) {
+      SMARTLIST_FOREACH(ns->routerstatus_list, vote_routerstatus_t *, rs,
+                        vote_routerstatus_free(rs));
+    } else {
+      SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *, rs,
+                        routerstatus_free(rs));
+    }
+
+    smartlist_free(ns->routerstatus_list);
+  }
+
+  digestmap_free(ns->desc_digest_map, NULL);
+
+  if (ns->sr_info.commits) {
+    dirvote_clear_commits(ns);
+  }
+  tor_free(ns->sr_info.previous_srv);
+  tor_free(ns->sr_info.current_srv);
+
+  memwipe(ns, 11, sizeof(*ns));
+  tor_free(ns);
+}
+
+/** Return the voter info from <b>vote</b> for the voter whose identity digest
+ * is <b>identity</b>, or NULL if no such voter is associated with
+ * <b>vote</b>. */
+networkstatus_voter_info_t *
+networkstatus_get_voter_by_id(networkstatus_t *vote,
+                              const char *identity)
+{
+  if (!vote || !vote->voters)
+    return NULL;
+  SMARTLIST_FOREACH(vote->voters, networkstatus_voter_info_t *, voter,
+    if (fast_memeq(voter->identity_digest, identity, DIGEST_LEN))
+      return voter);
+  return NULL;
+}
+
+/** Return the signature made by <b>voter</b> using the algorithm
+ * <b>alg</b>, or NULL if none is found. */
+document_signature_t *
+networkstatus_get_voter_sig_by_alg(const networkstatus_voter_info_t *voter,
+                                   digest_algorithm_t alg)
+{
+  if (!voter->sigs)
+    return NULL;
+  SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                    if (sig->alg == alg)
+                    return sig);
+  return NULL;
+}
+
+/** Check whether the signature <b>sig</b> is correctly signed with the
+ * signing key in <b>cert</b>.  Return -1 if <b>cert</b> doesn't match the
+ * signing key; otherwise set the good_signature or bad_signature flag on
+ * <b>voter</b>, and return 0. */
+int
+networkstatus_check_document_signature(const networkstatus_t *consensus,
+                                       document_signature_t *sig,
+                                       const authority_cert_t *cert)
+{
+  char key_digest[DIGEST_LEN];
+  const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN;
+  char *signed_digest;
+  size_t signed_digest_len;
+
+  if (crypto_pk_get_digest(cert->signing_key, key_digest)<0)
+    return -1;
+  if (tor_memneq(sig->signing_key_digest, key_digest, DIGEST_LEN) ||
+      tor_memneq(sig->identity_digest, cert->cache_info.identity_digest,
+                 DIGEST_LEN))
+    return -1;
+
+  if (authority_cert_is_blacklisted(cert)) {
+    /* We implement blacklisting for authority signing keys by treating
+     * all their signatures as always bad. That way we don't get into
+     * crazy loops of dropping and re-fetching signatures. */
+    log_warn(LD_DIR, "Ignoring a consensus signature made with deprecated"
+             " signing key %s",
+             hex_str(cert->signing_key_digest, DIGEST_LEN));
+    sig->bad_signature = 1;
+    return 0;
+  }
+
+  signed_digest_len = crypto_pk_keysize(cert->signing_key);
+  signed_digest = tor_malloc(signed_digest_len);
+  if (crypto_pk_public_checksig(cert->signing_key,
+                                signed_digest,
+                                signed_digest_len,
+                                sig->signature,
+                                sig->signature_len) < dlen ||
+      tor_memneq(signed_digest, consensus->digests.d[sig->alg], dlen)) {
+    log_warn(LD_DIR, "Got a bad signature on a networkstatus vote");
+    sig->bad_signature = 1;
+  } else {
+    sig->good_signature = 1;
+  }
+  tor_free(signed_digest);
+  return 0;
+}
+
+/** Given a v3 networkstatus consensus in <b>consensus</b>, check every
+ * as-yet-unchecked signature on <b>consensus</b>.  Return 1 if there is a
+ * signature from every recognized authority on it, 0 if there are
+ * enough good signatures from recognized authorities on it, -1 if we might
+ * get enough good signatures by fetching missing certificates, and -2
+ * otherwise.  Log messages at INFO or WARN: if <b>warn</b> is over 1, warn
+ * about every problem; if warn is at least 1, warn only if we can't get
+ * enough signatures; if warn is negative, log nothing at all. */
+int
+networkstatus_check_consensus_signature(networkstatus_t *consensus,
+                                        int warn)
+{
+  int n_good = 0;
+  int n_missing_key = 0, n_dl_failed_key = 0;
+  int n_bad = 0;
+  int n_unknown = 0;
+  int n_no_signature = 0;
+  int n_v3_authorities = get_n_authorities(V3_DIRINFO);
+  int n_required = n_v3_authorities/2 + 1;
+  smartlist_t *list_good = smartlist_new();
+  smartlist_t *list_no_signature = smartlist_new();
+  smartlist_t *need_certs_from = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  smartlist_t *missing_authorities = smartlist_new();
+  int severity;
+  time_t now = time(NULL);
+
+  tor_assert(consensus->type == NS_TYPE_CONSENSUS);
+
+  SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *,
+                          voter) {
+    int good_here = 0;
+    int bad_here = 0;
+    int unknown_here = 0;
+    int missing_key_here = 0, dl_failed_key_here = 0;
+    SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
+      if (!sig->good_signature && !sig->bad_signature &&
+          sig->signature) {
+        /* we can try to check the signature. */
+        int is_v3_auth = trusteddirserver_get_by_v3_auth_digest(
+                                              sig->identity_digest) != NULL;
+        authority_cert_t *cert =
+          authority_cert_get_by_digests(sig->identity_digest,
+                                        sig->signing_key_digest);
+        tor_assert(tor_memeq(sig->identity_digest, voter->identity_digest,
+                           DIGEST_LEN));
+
+        if (!is_v3_auth) {
+          smartlist_add(unrecognized, voter);
+          ++unknown_here;
+          continue;
+        } else if (!cert || cert->expires < now) {
+          smartlist_add(need_certs_from, voter);
+          ++missing_key_here;
+          if (authority_cert_dl_looks_uncertain(sig->identity_digest))
+            ++dl_failed_key_here;
+          continue;
+        }
+        if (networkstatus_check_document_signature(consensus, sig, cert) < 0) {
+          smartlist_add(need_certs_from, voter);
+          ++missing_key_here;
+          if (authority_cert_dl_looks_uncertain(sig->identity_digest))
+            ++dl_failed_key_here;
+          continue;
+        }
+      }
+      if (sig->good_signature)
+        ++good_here;
+      else if (sig->bad_signature)
+        ++bad_here;
+    } SMARTLIST_FOREACH_END(sig);
+
+    if (good_here) {
+      ++n_good;
+      smartlist_add(list_good, voter->nickname);
+    } else if (bad_here) {
+      ++n_bad;
+    } else if (missing_key_here) {
+      ++n_missing_key;
+      if (dl_failed_key_here)
+        ++n_dl_failed_key;
+    } else if (unknown_here) {
+      ++n_unknown;
+    } else {
+      ++n_no_signature;
+      smartlist_add(list_no_signature, voter->nickname);
+    }
+  } SMARTLIST_FOREACH_END(voter);
+
+  /* Now see whether we're missing any voters entirely. */
+  SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
+                    dir_server_t *, ds,
+    {
+      if ((ds->type & V3_DIRINFO) &&
+          !networkstatus_get_voter_by_id(consensus, ds->v3_identity_digest))
+        smartlist_add(missing_authorities, ds);
+    });
+
+  if (warn > 1 || (warn >= 0 &&
+                   (n_good + n_missing_key - n_dl_failed_key < n_required))) {
+    severity = LOG_WARN;
+  } else {
+    severity = LOG_INFO;
+  }
+
+  if (warn >= 0) {
+    SMARTLIST_FOREACH(unrecognized, networkstatus_voter_info_t *, voter,
+      {
+        tor_log(severity, LD_DIR, "Consensus includes unrecognized authority "
+                 "'%s' at %s:%d (contact %s; identity %s)",
+                 voter->nickname, voter->address, (int)voter->dir_port,
+                 voter->contact?voter->contact:"n/a",
+                 hex_str(voter->identity_digest, DIGEST_LEN));
+      });
+    SMARTLIST_FOREACH(need_certs_from, networkstatus_voter_info_t *, voter,
+      {
+        tor_log(severity, LD_DIR, "Looks like we need to download a new "
+                 "certificate from authority '%s' at %s:%d (contact %s; "
+                 "identity %s)",
+                 voter->nickname, voter->address, (int)voter->dir_port,
+                 voter->contact?voter->contact:"n/a",
+                 hex_str(voter->identity_digest, DIGEST_LEN));
+      });
+    SMARTLIST_FOREACH(missing_authorities, dir_server_t *, ds,
+      {
+        tor_log(severity, LD_DIR, "Consensus does not include configured "
+                 "authority '%s' at %s:%d (identity %s)",
+                 ds->nickname, ds->address, (int)ds->dir_port,
+                 hex_str(ds->v3_identity_digest, DIGEST_LEN));
+      });
+    {
+      char *joined;
+      smartlist_t *sl = smartlist_new();
+      char *tmp = smartlist_join_strings(list_good, " ", 0, NULL);
+      smartlist_add_asprintf(sl,
+                   "A consensus needs %d good signatures from recognized "
+                   "authorities for us to accept it. "
+                   "This %s one has %d (%s).",
+                   n_required,
+                   networkstatus_get_flavor_name(consensus->flavor),
+                   n_good, tmp);
+      tor_free(tmp);
+      if (n_no_signature) {
+        tmp = smartlist_join_strings(list_no_signature, " ", 0, NULL);
+        smartlist_add_asprintf(sl,
+                     "%d (%s) of the authorities we know didn't sign it.",
+                     n_no_signature, tmp);
+        tor_free(tmp);
+      }
+      if (n_unknown) {
+        smartlist_add_asprintf(sl,
+                      "It has %d signatures from authorities we don't "
+                      "recognize.", n_unknown);
+      }
+      if (n_bad) {
+        smartlist_add_asprintf(sl, "%d of the signatures on it didn't verify "
+                      "correctly.", n_bad);
+      }
+      if (n_missing_key) {
+        smartlist_add_asprintf(sl,
+                      "We were unable to check %d of the signatures, "
+                      "because we were missing the keys.", n_missing_key);
+      }
+      joined = smartlist_join_strings(sl, " ", 0, NULL);
+      tor_log(severity, LD_DIR, "%s", joined);
+      tor_free(joined);
+      SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+      smartlist_free(sl);
+    }
+  }
+
+  smartlist_free(list_good);
+  smartlist_free(list_no_signature);
+  smartlist_free(unrecognized);
+  smartlist_free(need_certs_from);
+  smartlist_free(missing_authorities);
+
+  if (n_good == n_v3_authorities)
+    return 1;
+  else if (n_good >= n_required)
+    return 0;
+  else if (n_good + n_missing_key >= n_required)
+    return -1;
+  else
+    return -2;
+}
+
+/** How far in the future do we allow a network-status to get before removing
+ * it? (seconds) */
+#define NETWORKSTATUS_ALLOW_SKEW (24*60*60)
+
+/** Helper for bsearching a list of routerstatus_t pointers: compare a
+ * digest in the key to the identity digest of a routerstatus_t. */
+int
+compare_digest_to_routerstatus_entry(const void *_key, const void **_member)
+{
+  const char *key = _key;
+  const routerstatus_t *rs = *_member;
+  return tor_memcmp(key, rs->identity_digest, DIGEST_LEN);
+}
+
+/** Helper for bsearching a list of routerstatus_t pointers: compare a
+ * digest in the key to the identity digest of a routerstatus_t. */
+int
+compare_digest_to_vote_routerstatus_entry(const void *_key,
+                                          const void **_member)
+{
+  const char *key = _key;
+  const vote_routerstatus_t *vrs = *_member;
+  return tor_memcmp(key, vrs->status.identity_digest, DIGEST_LEN);
+}
+
+/** As networkstatus_find_entry, but do not return a const pointer */
+routerstatus_t *
+networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest)
+{
+  return smartlist_bsearch(ns->routerstatus_list, digest,
+                           compare_digest_to_routerstatus_entry);
+}
+
+/** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or
+ * NULL if none was found. */
+const routerstatus_t *
+networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+{
+  return networkstatus_vote_find_mutable_entry(ns, digest);
+}
+
+/*XXXX MOVE make this static once functions are moved into this file. */
+/** Search the routerstatuses in <b>ns</b> for one whose identity digest is
+ * <b>digest</b>.  Return value and set *<b>found_out</b> as for
+ * smartlist_bsearch_idx(). */
+int
+networkstatus_vote_find_entry_idx(networkstatus_t *ns,
+                                  const char *digest, int *found_out)
+{
+  return smartlist_bsearch_idx(ns->routerstatus_list, digest,
+                               compare_digest_to_routerstatus_entry,
+                               found_out);
+}
+
+/** As router_get_consensus_status_by_descriptor_digest, but does not return
+ * a const pointer. */
+MOCK_IMPL(routerstatus_t *,
+router_get_mutable_consensus_status_by_descriptor_digest,(
+                                                 networkstatus_t *consensus,
+                                                 const char *digest))
+{
+  if (!consensus)
+    consensus = networkstatus_get_latest_consensus();
+  if (!consensus)
+    return NULL;
+  if (!consensus->desc_digest_map) {
+    digestmap_t *m = consensus->desc_digest_map = digestmap_new();
+    SMARTLIST_FOREACH(consensus->routerstatus_list,
+                      routerstatus_t *, rs,
+     {
+       digestmap_set(m, rs->descriptor_digest, rs);
+     });
+  }
+  return digestmap_get(consensus->desc_digest_map, digest);
+}
+
+/** Return the consensus view of the status of the router whose current
+ * <i>descriptor</i> digest in <b>consensus</b> is <b>digest</b>, or NULL if
+ * no such router is known. */
+const routerstatus_t *
+router_get_consensus_status_by_descriptor_digest(networkstatus_t *consensus,
+                                                 const char *digest)
+{
+  return router_get_mutable_consensus_status_by_descriptor_digest(
+                                                          consensus, digest);
+}
+
+/** Return a smartlist of all router descriptor digests in a consensus */
+static smartlist_t *
+router_get_descriptor_digests_in_consensus(networkstatus_t *consensus)
+{
+  smartlist_t *result = smartlist_new();
+  digestmap_iter_t *i;
+  const char *digest;
+  void *rs;
+  char *digest_tmp;
+
+  for (i = digestmap_iter_init(consensus->desc_digest_map);
+       !(digestmap_iter_done(i));
+       i = digestmap_iter_next(consensus->desc_digest_map, i)) {
+    digestmap_iter_get(i, &digest, &rs);
+    digest_tmp = tor_malloc(DIGEST_LEN);
+    memcpy(digest_tmp, digest, DIGEST_LEN);
+    smartlist_add(result, digest_tmp);
+  }
+
+  return result;
+}
+
+/** Return a smartlist of all router descriptor digests in the current
+ * consensus */
+MOCK_IMPL(smartlist_t *,
+router_get_descriptor_digests,(void))
+{
+  smartlist_t *result = NULL;
+
+  if (current_ns_consensus) {
+    result =
+      router_get_descriptor_digests_in_consensus(current_ns_consensus);
+  }
+
+  return result;
+}
+
+/** Given the digest of a router descriptor, return its current download
+ * status, or NULL if the digest is unrecognized. */
+MOCK_IMPL(download_status_t *,
+router_get_dl_status_by_descriptor_digest,(const char *d))
+{
+  routerstatus_t *rs;
+  if (!current_ns_consensus)
+    return NULL;
+  if ((rs = router_get_mutable_consensus_status_by_descriptor_digest(
+                                              current_ns_consensus, d)))
+    return &rs->dl_status;
+
+  return NULL;
+}
+
+/** As router_get_consensus_status_by_id, but do not return a const pointer */
+routerstatus_t *
+router_get_mutable_consensus_status_by_id(const char *digest)
+{
+  const networkstatus_t *ns = networkstatus_get_latest_consensus();
+  if (!ns)
+    return NULL;
+  smartlist_t *rslist = ns->routerstatus_list;
+  return smartlist_bsearch(rslist, digest,
+                           compare_digest_to_routerstatus_entry);
+}
+
+/** Return the consensus view of the status of the router whose identity
+ * digest is <b>digest</b>, or NULL if we don't know about any such router. */
+const routerstatus_t *
+router_get_consensus_status_by_id(const char *digest)
+{
+  return router_get_mutable_consensus_status_by_id(digest);
+}
+
+/** How frequently do directory authorities re-download fresh networkstatus
+ * documents? */
+#define AUTHORITY_NS_CACHE_INTERVAL (10*60)
+
+/** How frequently do non-authority directory caches re-download fresh
+ * networkstatus documents? */
+#define NONAUTHORITY_NS_CACHE_INTERVAL (60*60)
+
+/** Return true iff, given the options listed in <b>options</b>, <b>flavor</b>
+ *  is the flavor of a consensus networkstatus that we would like to fetch.
+ *
+ * For certificate fetches, use we_want_to_fetch_unknown_auth_certs, and
+ * for serving fetched documents, use directory_caches_dir_info. */
+int
+we_want_to_fetch_flavor(const or_options_t *options, int flavor)
+{
+  if (flavor < 0 || flavor > N_CONSENSUS_FLAVORS) {
+    /* This flavor is crazy; we don't want it */
+    /*XXXX handle unrecognized flavors later */
+    return 0;
+  }
+  if (authdir_mode_v3(options) || directory_caches_dir_info(options)) {
+    /* We want to serve all flavors to others, regardless if we would use
+     * it ourselves. */
+    return 1;
+  }
+  if (options->FetchUselessDescriptors) {
+    /* In order to get all descriptors, we need to fetch all consensuses. */
+    return 1;
+  }
+  /* Otherwise, we want the flavor only if we want to use it to build
+   * circuits. */
+  return flavor == usable_consensus_flavor();
+}
+
+/** Return true iff, given the options listed in <b>options</b>, we would like
+ * to fetch and store unknown authority certificates.
+ *
+ * For consensus and descriptor fetches, use we_want_to_fetch_flavor, and
+ * for serving fetched certificates, use directory_caches_unknown_auth_certs.
+ */
+int
+we_want_to_fetch_unknown_auth_certs(const or_options_t *options)
+{
+  if (authdir_mode_v3(options) ||
+      directory_caches_unknown_auth_certs((options))) {
+    /* We want to serve all certs to others, regardless if we would use
+     * them ourselves. */
+    return 1;
+  }
+  if (options->FetchUselessDescriptors) {
+    /* Unknown certificates are definitely useless. */
+    return 1;
+  }
+  /* Otherwise, don't fetch unknown certificates. */
+  return 0;
+}
+
+/** How long will we hang onto a possibly live consensus for which we're
+ * fetching certs before we check whether there is a better one? */
+#define DELAY_WHILE_FETCHING_CERTS (20*60)
+
+/** What is the minimum time we need to have waited fetching certs, before we
+ * increment the consensus download schedule on failure? */
+#define MIN_DELAY_FOR_FETCH_CERT_STATUS_FAILURE (1*60)
+
+/* Check if a downloaded consensus flavor should still wait for certificates
+ * to download now. If we decide not to wait, check if enough time has passed
+ * to consider the certificate download failure a separate failure. If so,
+ * fail dls.
+ * If waiting for certificates to download, return 1. If not, return 0. */
+static int
+check_consensus_waiting_for_certs(int flavor, time_t now,
+                                  download_status_t *dls)
+{
+  consensus_waiting_for_certs_t *waiting;
+
+  /* We should always have a known flavor, because we_want_to_fetch_flavor()
+   * filters out unknown flavors. */
+  tor_assert(flavor >= 0 && flavor < N_CONSENSUS_FLAVORS);
+
+  waiting = &consensus_waiting_for_certs[flavor];
+  if (waiting->consensus) {
+    /* XXXX make sure this doesn't delay sane downloads. */
+    if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now &&
+        waiting->consensus->valid_until > now) {
+      return 1;
+    } else {
+      if (!waiting->dl_failed) {
+        if (waiting->set_at + MIN_DELAY_FOR_FETCH_CERT_STATUS_FAILURE > now) {
+          download_status_failed(dls, 0);
+        }
+        waiting->dl_failed=1;
+      }
+    }
+  }
+
+  return 0;
+}
+
+/** If we want to download a fresh consensus, launch a new download as
+ * appropriate. */
+static void
+update_consensus_networkstatus_downloads(time_t now)
+{
+  int i;
+  const or_options_t *options = get_options();
+  const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping(
+                                                                        now);
+  const int use_multi_conn =
+    networkstatus_consensus_can_use_multiple_directories(options);
+
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
+    /* XXXX need some way to download unknown flavors if we are caching. */
+    const char *resource;
+    networkstatus_t *c;
+    int max_in_progress_conns = 1;
+
+    if (! we_want_to_fetch_flavor(options, i))
+      continue;
+
+    c = networkstatus_get_latest_consensus_by_flavor(i);
+    if (! (c && c->valid_after <= now && now <= c->valid_until)) {
+      /* No live consensus? Get one now!*/
+      time_to_download_next_consensus[i] = now;
+    }
+
+    if (time_to_download_next_consensus[i] > now)
+      continue; /* Wait until the current consensus is older. */
+
+    resource = networkstatus_get_flavor_name(i);
+
+    /* Check if we already have enough connections in progress */
+    if (we_are_bootstrapping && use_multi_conn) {
+      max_in_progress_conns =
+        options->ClientBootstrapConsensusMaxInProgressTries;
+    }
+    if (connection_dir_count_by_purpose_and_resource(
+                                                  DIR_PURPOSE_FETCH_CONSENSUS,
+                                                  resource)
+        >= max_in_progress_conns) {
+      continue;
+    }
+
+    /* Check if we want to launch another download for a usable consensus.
+     * Only used during bootstrap. */
+    if (we_are_bootstrapping && use_multi_conn
+        && i == usable_consensus_flavor()) {
+
+      /* Check if we're already downloading a usable consensus */
+      if (networkstatus_consensus_is_already_downloading(resource))
+        continue;
+
+      /* Make multiple connections for a bootstrap consensus download. */
+      update_consensus_bootstrap_multiple_downloads(now, options);
+    } else {
+      /* Check if we failed downloading a consensus too recently */
+
+      /* Let's make sure we remembered to update consensus_dl_status */
+      tor_assert(consensus_dl_status[i].schedule == DL_SCHED_CONSENSUS);
+
+      if (!download_status_is_ready(&consensus_dl_status[i], now)) {
+        continue;
+      }
+
+      /** Check if we're waiting for certificates to download. If we are,
+       * launch download for missing directory authority certificates. */
+      if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i])) {
+        update_certificate_downloads(now);
+        continue;
+      }
+
+      /* Try the requested attempt */
+      log_info(LD_DIR, "Launching %s standard networkstatus consensus "
+               "download.", networkstatus_get_flavor_name(i));
+      directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
+                                   ROUTER_PURPOSE_GENERAL, resource,
+                                   PDS_RETRY_IF_NO_SERVERS,
+                                   consensus_dl_status[i].want_authority);
+    }
+  }
+}
+
+/** When we're bootstrapping, launch one or more consensus download
+ * connections, if schedule indicates connection(s) should be made after now.
+ * If is_authority, connect to an authority, otherwise, use a fallback
+ * directory mirror.
+ */
+static void
+update_consensus_bootstrap_attempt_downloads(
+                                      time_t now,
+                                      download_status_t *dls,
+                                      download_want_authority_t want_authority)
+{
+  const char *resource = networkstatus_get_flavor_name(
+                                                  usable_consensus_flavor());
+
+  /* Let's make sure we remembered to update schedule */
+  tor_assert(dls->schedule == DL_SCHED_CONSENSUS);
+
+  /* Allow for multiple connections in the same second, if the schedule value
+   * is 0. */
+  while (download_status_is_ready(dls, now)) {
+    log_info(LD_DIR, "Launching %s bootstrap %s networkstatus consensus "
+             "download.", resource, (want_authority == DL_WANT_AUTHORITY
+                                     ? "authority"
+                                     : "mirror"));
+
+    directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
+                                 ROUTER_PURPOSE_GENERAL, resource,
+                                 PDS_RETRY_IF_NO_SERVERS, want_authority);
+    /* schedule the next attempt */
+    download_status_increment_attempt(dls, resource, now);
+  }
+}
+
+/** If we're bootstrapping, check the connection schedules and see if we want
+ * to make additional, potentially concurrent, consensus download
+ * connections.
+ * Only call when bootstrapping, and when we want to make additional
+ * connections. Only nodes that satisfy
+ * networkstatus_consensus_can_use_multiple_directories make additional
+ * connections.
+ */
+static void
+update_consensus_bootstrap_multiple_downloads(time_t now,
+                                              const or_options_t *options)
+{
+  const int usable_flavor = usable_consensus_flavor();
+
+  /* make sure we can use multiple connections */
+  if (!networkstatus_consensus_can_use_multiple_directories(options)) {
+    return;
+  }
+
+  /* Launch concurrent consensus download attempt(s) based on the mirror and
+   * authority schedules. Try the mirror first - this makes it slightly more
+   * likely that we'll connect to the fallback first, and then end the
+   * authority connection attempt. */
+
+  /* If a consensus download fails because it's waiting for certificates,
+   * we'll fail both the authority and fallback schedules. This is better than
+   * failing only one of the schedules, and having the other continue
+   * unchecked.
+   */
+
+  /* If we don't have or can't use extra fallbacks, don't try them. */
+  if (networkstatus_consensus_can_use_extra_fallbacks(options)) {
+    download_status_t *dls_f =
+      &consensus_bootstrap_dl_status[CONSENSUS_BOOTSTRAP_SOURCE_ANY_DIRSERVER];
+
+    if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_f)) {
+      /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */
+      update_consensus_bootstrap_attempt_downloads(now, dls_f,
+                                                   DL_WANT_ANY_DIRSERVER);
+    }
+  }
+
+  /* Now try an authority. */
+  download_status_t *dls_a =
+    &consensus_bootstrap_dl_status[CONSENSUS_BOOTSTRAP_SOURCE_AUTHORITY];
+
+  if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_a)) {
+    update_consensus_bootstrap_attempt_downloads(now, dls_a,
+                                                 DL_WANT_AUTHORITY);
+  }
+}
+
+/** Called when an attempt to download a consensus fails: note that the
+ * failure occurred, and possibly retry. */
+void
+networkstatus_consensus_download_failed(int status_code, const char *flavname)
+{
+  int flav = networkstatus_parse_flavor_name(flavname);
+  if (flav >= 0) {
+    tor_assert(flav < N_CONSENSUS_FLAVORS);
+    /* XXXX handle unrecognized flavors */
+    download_status_failed(&consensus_dl_status[flav], status_code);
+    /* Retry immediately, if appropriate. */
+    update_consensus_networkstatus_downloads(time(NULL));
+  }
+}
+
+/** How long do we (as a cache) wait after a consensus becomes non-fresh
+ * before trying to fetch another? */
+#define CONSENSUS_MIN_SECONDS_BEFORE_CACHING 120
+
+/** Update the time at which we'll consider replacing the current
+ * consensus of flavor <b>flav</b> */
+static void
+update_consensus_networkstatus_fetch_time_impl(time_t now, int flav)
+{
+  const or_options_t *options = get_options();
+  networkstatus_t *c = networkstatus_get_latest_consensus_by_flavor(flav);
+  const char *flavor = networkstatus_get_flavor_name(flav);
+  if (! we_want_to_fetch_flavor(get_options(), flav))
+    return;
+
+  if (c && c->valid_after <= now && now <= c->valid_until) {
+    long dl_interval;
+    long interval = c->fresh_until - c->valid_after;
+    long min_sec_before_caching = CONSENSUS_MIN_SECONDS_BEFORE_CACHING;
+    time_t start;
+
+    if (min_sec_before_caching > interval/16) {
+      /* Usually we allow 2-minutes slop factor in case clocks get
+         desynchronized a little.  If we're on a private network with
+         a crazy-fast voting interval, though, 2 minutes may be too
+         much. */
+      min_sec_before_caching = interval/16;
+      /* make sure we always delay by at least a second before caching */
+      if (min_sec_before_caching == 0) {
+        min_sec_before_caching = 1;
+      }
+    }
+
+    if (directory_fetches_dir_info_early(options)) {
+      /* We want to cache the next one at some point after this one
+       * is no longer fresh... */
+      start = (time_t)(c->fresh_until + min_sec_before_caching);
+      /* Some clients may need the consensus sooner than others. */
+      if (options->FetchDirInfoExtraEarly || authdir_mode_v3(options)) {
+        dl_interval = 60;
+        if (min_sec_before_caching + dl_interval > interval)
+          dl_interval = interval/2;
+      } else {
+        /* But only in the first half-interval after that. */
+        dl_interval = interval/2;
+      }
+    } else {
+      /* We're an ordinary client, a bridge, or a hidden service.
+       * Give all the caches enough time to download the consensus. */
+      start = (time_t)(c->fresh_until + (interval*3)/4);
+      /* But download the next one well before this one is expired. */
+      dl_interval = ((c->valid_until - start) * 7 )/ 8;
+
+      /* If we're a bridge user, make use of the numbers we just computed
+       * to choose the rest of the interval *after* them. */
+      if (directory_fetches_dir_info_later(options)) {
+        /* Give all the *clients* enough time to download the consensus. */
+        start = (time_t)(start + dl_interval + min_sec_before_caching);
+        /* But try to get it before ours actually expires. */
+        dl_interval = (c->valid_until - start) - min_sec_before_caching;
+      }
+    }
+    /* catch low dl_interval in crazy-fast networks */
+    if (dl_interval < 1)
+      dl_interval = 1;
+    /* catch late start in crazy-fast networks */
+    if (start+dl_interval >= c->valid_until)
+      start = c->valid_until - dl_interval - 1;
+    log_debug(LD_DIR,
+              "fresh_until: %ld start: %ld "
+              "dl_interval: %ld valid_until: %ld ",
+              (long)c->fresh_until, (long)start, dl_interval,
+              (long)c->valid_until);
+    /* We must not try to replace c while it's still fresh: */
+    tor_assert(c->fresh_until < start);
+    /* We must download the next one before c is invalid: */
+    tor_assert(start+dl_interval < c->valid_until);
+    time_to_download_next_consensus[flav] =
+      start + crypto_rand_int((int)dl_interval);
+    {
+      char tbuf1[ISO_TIME_LEN+1];
+      char tbuf2[ISO_TIME_LEN+1];
+      char tbuf3[ISO_TIME_LEN+1];
+      format_local_iso_time(tbuf1, c->fresh_until);
+      format_local_iso_time(tbuf2, c->valid_until);
+      format_local_iso_time(tbuf3, time_to_download_next_consensus[flav]);
+      log_info(LD_DIR, "Live %s consensus %s the most recent until %s and "
+               "will expire at %s; fetching the next one at %s.",
+               flavor, (c->fresh_until > now) ? "will be" : "was",
+               tbuf1, tbuf2, tbuf3);
+    }
+  } else {
+    time_to_download_next_consensus[flav] = now;
+    log_info(LD_DIR, "No live %s consensus; we should fetch one immediately.",
+             flavor);
+  }
+}
+
+/** Update the time at which we'll consider replacing the current
+ * consensus of flavor 'flavor' */
+void
+update_consensus_networkstatus_fetch_time(time_t now)
+{
+  int i;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    if (we_want_to_fetch_flavor(get_options(), i))
+      update_consensus_networkstatus_fetch_time_impl(now, i);
+  }
+}
+
+/** Return 1 if there's a reason we shouldn't try any directory
+ * fetches yet (e.g. we demand bridges and none are yet known).
+ * Else return 0.
+
+ * If we return 1 and <b>msg_out</b> is provided, set <b>msg_out</b>
+ * to an explanation of why directory fetches are delayed. (If we
+ * return 0, we set msg_out to NULL.)
+ */
+int
+should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
+{
+  if (msg_out) {
+    *msg_out = NULL;
+  }
+
+  if (options->DisableNetwork) {
+    if (msg_out) {
+      *msg_out = "DisableNetwork is set.";
+    }
+    log_info(LD_DIR, "Delaying dir fetches (DisableNetwork is set)");
+    return 1;
+  }
+
+  if (we_are_hibernating()) {
+    if (msg_out) {
+      *msg_out = "We are hibernating or shutting down.";
+    }
+    log_info(LD_DIR, "Delaying dir fetches (Hibernating or shutting down)");
+    return 1;
+  }
+
+  if (options->UseBridges) {
+    /* If we know that none of our bridges can possibly work, avoid fetching
+     * directory documents. But if some of them might work, try again. */
+    if (num_bridges_usable(1) == 0) {
+      if (msg_out) {
+        *msg_out = "No running bridges";
+      }
+      log_info(LD_DIR, "Delaying dir fetches (no running bridges known)");
+      return 1;
+    }
+
+    if (pt_proxies_configuration_pending()) {
+      if (msg_out) {
+        *msg_out = "Pluggable transport proxies still configuring";
+      }
+      log_info(LD_DIR, "Delaying dir fetches (pt proxies still configuring)");
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+/** Launch requests for networkstatus documents as appropriate. This is called
+ * when we retry all the connections on a SIGHUP and periodically by a Periodic
+ * event which checks whether we want to download any networkstatus documents.
+ */
+void
+update_networkstatus_downloads(time_t now)
+{
+  const or_options_t *options = get_options();
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+  /** Launch a consensus download request, we will wait for the consensus to
+   * download and when it completes we will launch a certificate download
+   * request. */
+  update_consensus_networkstatus_downloads(now);
+}
+
+/** Launch requests as appropriate for missing directory authority
+ * certificates. */
+void
+update_certificate_downloads(time_t now)
+{
+  int i;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    if (consensus_waiting_for_certs[i].consensus)
+      authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus,
+                                    now, NULL);
+  }
+
+  if (current_ns_consensus)
+    authority_certs_fetch_missing(current_ns_consensus, now, NULL);
+  if (current_md_consensus)
+    authority_certs_fetch_missing(current_md_consensus, now, NULL);
+}
+
+/** Return 1 if we have a consensus but we don't have enough certificates
+ * to start using it yet. */
+int
+consensus_is_waiting_for_certs(void)
+{
+  return consensus_waiting_for_certs[usable_consensus_flavor()].consensus
+    ? 1 : 0;
+}
+
+/** Look up the currently active (depending on bootstrap status) download
+ * status for this consensus flavor and return a pointer to it.
+ */
+MOCK_IMPL(download_status_t *,
+networkstatus_get_dl_status_by_flavor,(consensus_flavor_t flavor))
+{
+  download_status_t *dl = NULL;
+  const int we_are_bootstrapping =
+    networkstatus_consensus_is_bootstrapping(time(NULL));
+
+  if ((int)flavor <= N_CONSENSUS_FLAVORS) {
+    dl = &((we_are_bootstrapping ?
+           consensus_bootstrap_dl_status : consensus_dl_status)[flavor]);
+  }
+
+  return dl;
+}
+
+/** Look up the bootstrap download status for this consensus flavor
+ * and return a pointer to it. */
+MOCK_IMPL(download_status_t *,
+networkstatus_get_dl_status_by_flavor_bootstrap,(consensus_flavor_t flavor))
+{
+  download_status_t *dl = NULL;
+
+  if ((int)flavor <= N_CONSENSUS_FLAVORS) {
+    dl = &(consensus_bootstrap_dl_status[flavor]);
+  }
+
+  return dl;
+}
+
+/** Look up the running (non-bootstrap) download status for this consensus
+ * flavor and return a pointer to it. */
+MOCK_IMPL(download_status_t *,
+networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor))
+{
+  download_status_t *dl = NULL;
+
+  if ((int)flavor <= N_CONSENSUS_FLAVORS) {
+    dl = &(consensus_dl_status[flavor]);
+  }
+
+  return dl;
+}
+
+/** Return the most recent consensus that we have downloaded, or NULL if we
+ * don't have one. */
+MOCK_IMPL(networkstatus_t *,
+networkstatus_get_latest_consensus,(void))
+{
+  if (we_use_microdescriptors_for_circuits(get_options()))
+    return current_md_consensus;
+  else
+    return current_ns_consensus;
+}
+
+/** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL
+ * if we don't have one. */
+MOCK_IMPL(networkstatus_t *,
+networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f))
+{
+  if (f == FLAV_NS)
+    return current_ns_consensus;
+  else if (f == FLAV_MICRODESC)
+    return current_md_consensus;
+  else {
+    tor_assert(0);
+    return NULL;
+  }
+}
+
+/** Return the most recent consensus that we have downloaded, or NULL if it is
+ * no longer live. */
+MOCK_IMPL(networkstatus_t *,
+networkstatus_get_live_consensus,(time_t now))
+{
+  networkstatus_t *ns = networkstatus_get_latest_consensus();
+  if (ns && networkstatus_is_live(ns, now))
+    return ns;
+  else
+    return NULL;
+}
+
+/** Given a consensus in <b>ns</b>, return true iff currently live and
+ *  unexpired. */
+int
+networkstatus_is_live(const networkstatus_t *ns, time_t now)
+{
+  return (ns->valid_after <= now && now <= ns->valid_until);
+}
+
+/** Determine if <b>consensus</b> is valid or expired recently enough that
+ * we can still use it.
+ *
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ */
+int
+networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                        time_t now)
+{
+  if (BUG(!consensus))
+    return 0;
+
+  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+                                                      now);
+}
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_until
+ * time rather than an entire consensus. */
+int
+networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                             time_t now)
+{
+#define REASONABLY_LIVE_TIME (24*60*60)
+  return (now <= valid_until + REASONABLY_LIVE_TIME);
+}
+
+/* XXXX remove this in favor of get_live_consensus. But actually,
+ * leave something like it for bridge users, who need to not totally
+ * lose if they spend a while fetching a new consensus. */
+/** As networkstatus_get_live_consensus(), but is way more tolerant of expired
+ * consensuses. */
+networkstatus_t *
+networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
+{
+  networkstatus_t *consensus =
+    networkstatus_get_latest_consensus_by_flavor(flavor);
+  if (consensus &&
+      consensus->valid_after <= now &&
+      networkstatus_consensus_reasonably_live(consensus, now))
+    return consensus;
+  else
+    return NULL;
+}
+
+/** Check if we need to download a consensus during tor's bootstrap phase.
+ * If we have no consensus, or our consensus is unusably old, return 1.
+ * As soon as we have received a consensus, return 0, even if we don't have
+ * enough certificates to validate it.
+ * If a fallback directory gives us a consensus we can never get certs for,
+ * check_consensus_waiting_for_certs() will wait 20 minutes before failing
+ * the cert downloads. After that, a new consensus will be fetched from a
+ * randomly chosen fallback. */
+MOCK_IMPL(int,
+networkstatus_consensus_is_bootstrapping,(time_t now))
+{
+  /* If we have a validated, reasonably live consensus, we're not
+   * bootstrapping a consensus at all. */
+  if (networkstatus_get_reasonably_live_consensus(
+                                                now,
+                                                usable_consensus_flavor())) {
+    return 0;
+  }
+
+  /* If we have a consensus, but we're waiting for certificates,
+   * we're not waiting for a consensus download while bootstrapping. */
+  if (consensus_is_waiting_for_certs()) {
+    return 0;
+  }
+
+  /* If we have no consensus, or our consensus is very old, we are
+   * bootstrapping, and we need to download a consensus. */
+  return 1;
+}
+
+/** Check if we can use multiple directories for a consensus download.
+ * Only clients (including bridge relays, which act like clients) benefit
+ * from multiple simultaneous consensus downloads. */
+int
+networkstatus_consensus_can_use_multiple_directories(
+                                                  const or_options_t *options)
+{
+  /* If we are a client, bridge, bridge client, or hidden service */
+  return !public_server_mode(options);
+}
+
+/** Check if we can use fallback directory mirrors for a consensus download.
+ * If we have fallbacks and don't want to fetch from the authorities,
+ * we can use them. */
+MOCK_IMPL(int,
+networkstatus_consensus_can_use_extra_fallbacks,(const or_options_t *options))
+{
+  /* The list length comparisons are a quick way to check if we have any
+   * non-authority fallback directories. If we ever have any authorities that
+   * aren't fallback directories, we will need to change this code. */
+  tor_assert(smartlist_len(router_get_fallback_dir_servers())
+             >= smartlist_len(router_get_trusted_dir_servers()));
+  /* If we don't fetch from the authorities, and we have additional mirrors,
+   * we can use them. */
+  return (!directory_fetches_from_authorities(options)
+          && (smartlist_len(router_get_fallback_dir_servers())
+              > smartlist_len(router_get_trusted_dir_servers())));
+}
+
+/* Is there a consensus fetch for flavor <b>resource</b> that's far
+ * enough along to be attached to a circuit? */
+int
+networkstatus_consensus_is_already_downloading(const char *resource)
+{
+  int answer = 0;
+
+  /* First, get a list of all the dir conns that are fetching a consensus,
+   * fetching *this* consensus, and are in state "reading" (meaning they
+   * have already flushed their request onto the socks connection). */
+  smartlist_t *fetching_conns =
+    connection_dir_list_by_purpose_resource_and_state(
+      DIR_PURPOSE_FETCH_CONSENSUS, resource, DIR_CONN_STATE_CLIENT_READING);
+
+  /* Then, walk through each conn, to see if its linked socks connection
+   * is in an attached state. We have to check this separately, since with
+   * the optimistic data feature, fetches can send their request to the
+   * socks connection and go into state 'reading', even before they're
+   * attached to any circuit. */
+  SMARTLIST_FOREACH_BEGIN(fetching_conns, dir_connection_t *, dirconn) {
+    /* Do any of these other dir conns have a linked socks conn that is
+     * attached to a circuit already? */
+    connection_t *base = TO_CONN(dirconn);
+    if (base->linked_conn &&
+        base->linked_conn->type == CONN_TYPE_AP &&
+        !AP_CONN_STATE_IS_UNATTACHED(base->linked_conn->state)) {
+      answer = 1;
+      break; /* stop looping, because we know the answer will be yes */
+    }
+  } SMARTLIST_FOREACH_END(dirconn);
+  smartlist_free(fetching_conns);
+
+  return answer;
+}
+
+/* Does the current, reasonably live consensus have IPv6 addresses?
+ * Returns 1 if there is a reasonably live consensus and its consensus method
+ * includes IPv6 addresses in the consensus.
+ * Otherwise, if there is no consensus, or the method does not include IPv6
+ * addresses, returns 0. */
+int
+networkstatus_consensus_has_ipv6(const or_options_t* options)
+{
+  const networkstatus_t *cons = networkstatus_get_reasonably_live_consensus(
+                                                    approx_time(),
+                                                    usable_consensus_flavor());
+
+  /* If we have no consensus, we have no IPv6 in it */
+  if (!cons) {
+    return 0;
+  }
+
+  /* Different flavours of consensus gained IPv6 at different times */
+  if (we_use_microdescriptors_for_circuits(options)) {
+    return
+       cons->consensus_method >= MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
+  } else {
+    return 1;
+  }
+}
+
+/** Given two router status entries for the same router identity, return 1 if
+ * if the contents have changed between them. Otherwise, return 0. */
+static int
+routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b)
+{
+  tor_assert(tor_memeq(a->identity_digest, b->identity_digest, DIGEST_LEN));
+
+  return strcmp(a->nickname, b->nickname) ||
+         fast_memneq(a->descriptor_digest, b->descriptor_digest, DIGEST_LEN) ||
+         a->addr != b->addr ||
+         a->or_port != b->or_port ||
+         a->dir_port != b->dir_port ||
+         a->is_authority != b->is_authority ||
+         a->is_exit != b->is_exit ||
+         a->is_stable != b->is_stable ||
+         a->is_fast != b->is_fast ||
+         a->is_flagged_running != b->is_flagged_running ||
+         a->is_named != b->is_named ||
+         a->is_unnamed != b->is_unnamed ||
+         a->is_valid != b->is_valid ||
+         a->is_possible_guard != b->is_possible_guard ||
+         a->is_bad_exit != b->is_bad_exit ||
+         a->is_hs_dir != b->is_hs_dir;
+  // XXXX this function needs a huge refactoring; it has gotten out
+  // XXXX of sync with routerstatus_t, and it will do so again.
+}
+
+/** Notify controllers of any router status entries that changed between
+ * <b>old_c</b> and <b>new_c</b>. */
+static void
+notify_control_networkstatus_changed(const networkstatus_t *old_c,
+                                     const networkstatus_t *new_c)
+{
+  smartlist_t *changed;
+  if (old_c == new_c)
+    return;
+
+  /* tell the controller exactly which relays are still listed, as well
+   * as what they're listed as */
+  control_event_newconsensus(new_c);
+
+  if (!control_event_is_interesting(EVENT_NS))
+    return;
+
+  if (!old_c) {
+    control_event_networkstatus_changed(new_c->routerstatus_list);
+    return;
+  }
+  changed = smartlist_new();
+
+  SMARTLIST_FOREACH_JOIN(
+                     old_c->routerstatus_list, const routerstatus_t *, rs_old,
+                     new_c->routerstatus_list, const routerstatus_t *, rs_new,
+                     tor_memcmp(rs_old->identity_digest,
+                            rs_new->identity_digest, DIGEST_LEN),
+                     smartlist_add(changed, (void*) rs_new)) {
+    if (routerstatus_has_changed(rs_old, rs_new))
+      smartlist_add(changed, (void*)rs_new);
+  } SMARTLIST_FOREACH_JOIN_END(rs_old, rs_new);
+
+  control_event_networkstatus_changed(changed);
+  smartlist_free(changed);
+}
+
+/* Called before the consensus changes from old_c to new_c. */
+static void
+notify_before_networkstatus_changes(const networkstatus_t *old_c,
+                                    const networkstatus_t *new_c)
+{
+  notify_control_networkstatus_changed(old_c, new_c);
+  dos_consensus_has_changed(new_c);
+  relay_consensus_has_changed(new_c);
+}
+
+/* Called after a new consensus has been put in the global state. It is safe
+ * to use the consensus getters in this function. */
+static void
+notify_after_networkstatus_changes(void)
+{
+  scheduler_notify_networkstatus_changed();
+}
+
+/** Copy all the ancillary information (like router download status and so on)
+ * from <b>old_c</b> to <b>new_c</b>. */
+static void
+networkstatus_copy_old_consensus_info(networkstatus_t *new_c,
+                                      const networkstatus_t *old_c)
+{
+  if (old_c == new_c)
+    return;
+  if (!old_c || !smartlist_len(old_c->routerstatus_list))
+    return;
+
+  SMARTLIST_FOREACH_JOIN(old_c->routerstatus_list, routerstatus_t *, rs_old,
+                         new_c->routerstatus_list, routerstatus_t *, rs_new,
+                         tor_memcmp(rs_old->identity_digest,
+                                rs_new->identity_digest, DIGEST_LEN),
+                         STMT_NIL) {
+    /* Okay, so we're looking at the same identity. */
+    rs_new->last_dir_503_at = rs_old->last_dir_503_at;
+
+    if (tor_memeq(rs_old->descriptor_digest, rs_new->descriptor_digest,
+                  DIGEST256_LEN)) {
+      /* And the same descriptor too! */
+      memcpy(&rs_new->dl_status, &rs_old->dl_status,sizeof(download_status_t));
+    }
+  } SMARTLIST_FOREACH_JOIN_END(rs_old, rs_new);
+}
+
+#ifdef TOR_UNIT_TESTS
+/**Accept a <b>flavor</b> consensus <b>c</b> without any additional
+ * validation. This is exclusively for unit tests.
+ * We copy any ancillary information from a pre-existing consensus
+ * and then free the current one and replace it with the newly
+ * provided instance. Returns -1 on unrecognized flavor, 0 otherwise.
+ */
+int
+networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
+                                            const char *flavor)
+{
+  int flav = networkstatus_parse_flavor_name(flavor);
+  switch (flav) {
+    case FLAV_NS:
+      if (current_ns_consensus) {
+        networkstatus_copy_old_consensus_info(c, current_ns_consensus);
+        networkstatus_vote_free(current_ns_consensus);
+      }
+      current_ns_consensus = c;
+      break;
+    case FLAV_MICRODESC:
+      if (current_md_consensus) {
+        networkstatus_copy_old_consensus_info(c, current_md_consensus);
+        networkstatus_vote_free(current_md_consensus);
+      }
+      current_md_consensus = c;
+      break;
+  }
+  return current_md_consensus ? 0 : -1;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Helper for handle_missing_protocol_warning: handles either the
+ * client case (if <b>is_client</b> is set) or the server case otherwise.
+ */
+static void
+handle_missing_protocol_warning_impl(const networkstatus_t *c,
+                                     int is_client)
+{
+  char *protocol_warning = NULL;
+
+  int should_exit = networkstatus_check_required_protocols(c,
+                                                   is_client,
+                                                   &protocol_warning);
+  if (protocol_warning) {
+    tor_log(should_exit ? LOG_ERR : LOG_WARN,
+            LD_GENERAL,
+            "%s", protocol_warning);
+  }
+  if (should_exit) {
+    tor_assert_nonfatal(protocol_warning);
+  }
+  tor_free(protocol_warning);
+  if (should_exit)
+    exit(1); // XXXX bad exit: should return from main.
+}
+
+/** Called when we have received a networkstatus <b>c</b>. If there are
+ * any _required_ protocols we are missing, log an error and exit
+ * immediately. If there are any _recommended_ protocols we are missing,
+ * warn. */
+static void
+handle_missing_protocol_warning(const networkstatus_t *c,
+                                const or_options_t *options)
+{
+  const int is_server = server_mode(options);
+  const int is_client = options_any_client_port_set(options) || !is_server;
+
+  if (is_server)
+    handle_missing_protocol_warning_impl(c, 0);
+  if (is_client)
+    handle_missing_protocol_warning_impl(c, 1);
+}
+
+/**
+ * Check whether we received a consensus that appears to be coming
+ * from the future.  Because we implicitly trust the directory
+ * authorities' idea of the current time, we produce a warning if we
+ * get an early consensus.
+ *
+ * If we got a consensus that is time stamped far in the past, that
+ * could simply have come from a stale cache.  Possible ways to get a
+ * consensus from the future can include:
+ *
+ * - enough directory authorities have wrong clocks
+ * - directory authorities collude to produce misleading time stamps
+ * - our own clock is wrong (this is by far the most likely)
+ *
+ * We neglect highly improbable scenarios that involve actual time
+ * travel.
+ */
+STATIC void
+warn_early_consensus(const networkstatus_t *c, const char *flavor,
+                     time_t now)
+{
+  char tbuf[ISO_TIME_LEN+1];
+  char dbuf[64];
+  long delta = now - c->valid_after;
+  char *flavormsg = NULL;
+
+/** If a consensus appears more than this many seconds before it could
+ * possibly be a sufficiently-signed consensus, declare that our clock
+ * is skewed. */
+#define EARLY_CONSENSUS_NOTICE_SKEW 60
+
+  /* We assume that if a majority of dirauths have accurate clocks,
+   * the earliest that a dirauth with a skewed clock could possibly
+   * publish a sufficiently-signed consensus is (valid_after -
+   * dist_seconds).  Before that time, the skewed dirauth would be
+   * unable to obtain enough authority signatures for the consensus to
+   * be valid. */
+  if (now >= c->valid_after - c->dist_seconds - EARLY_CONSENSUS_NOTICE_SKEW)
+    return;
+
+  format_iso_time(tbuf, c->valid_after);
+  format_time_interval(dbuf, sizeof(dbuf), delta);
+  log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
+           "consensus network status document (%s UTC).  Tor needs an "
+           "accurate clock to work correctly. Please check your time and "
+           "date settings!", dbuf, tbuf);
+  tor_asprintf(&flavormsg, "%s flavor consensus", flavor);
+  clock_skew_warning(NULL, delta, 1, LD_GENERAL, flavormsg, "CONSENSUS");
+  tor_free(flavormsg);
+}
+
+/** Try to replace the current cached v3 networkstatus with the one in
+ * <b>consensus</b>.  If we don't have enough certificates to validate it,
+ * store it in consensus_waiting_for_certs and launch a certificate fetch.
+ *
+ * If flags & NSSET_FROM_CACHE, this networkstatus has come from the disk
+ * cache.  If flags & NSSET_WAS_WAITING_FOR_CERTS, this networkstatus was
+ * already received, but we were waiting for certificates on it.  If flags &
+ * NSSET_DONT_DOWNLOAD_CERTS, do not launch certificate downloads as needed.
+ * If flags & NSSET_ACCEPT_OBSOLETE, then we should be willing to take this
+ * consensus, even if it comes from many days in the past.
+ *
+ * If source_dir is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved a consensus or certificates from, so try
+ * it first to fetch any missing certificates.
+ *
+ * Return 0 on success, <0 on failure.  On failure, caller should increment
+ * the failure count as appropriate.
+ *
+ * We return -1 for mild failures that don't need to be reported to the
+ * user, and -2 for more serious problems.
+ */
+int
+networkstatus_set_current_consensus(const char *consensus,
+                                    const char *flavor,
+                                    unsigned flags,
+                                    const char *source_dir)
+{
+  networkstatus_t *c=NULL;
+  int r, result = -1;
+  time_t now = approx_time();
+  const or_options_t *options = get_options();
+  char *unverified_fname = NULL, *consensus_fname = NULL;
+  int flav = networkstatus_parse_flavor_name(flavor);
+  const unsigned from_cache = flags & NSSET_FROM_CACHE;
+  const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS;
+  const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS);
+  const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE;
+  const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR;
+  const common_digests_t *current_digests = NULL;
+  consensus_waiting_for_certs_t *waiting = NULL;
+  time_t current_valid_after = 0;
+  int free_consensus = 1; /* Free 'c' at the end of the function */
+  int checked_protocols_already = 0;
+
+  if (flav < 0) {
+    /* XXXX we don't handle unrecognized flavors yet. */
+    log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor);
+    return -2;
+  }
+
+  /* Make sure it's parseable. */
+  c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS);
+  if (!c) {
+    log_warn(LD_DIR, "Unable to parse networkstatus consensus");
+    result = -2;
+    goto done;
+  }
+
+  if (from_cache && !was_waiting_for_certs) {
+    /* We previously stored this; check _now_ to make sure that version-kills
+     * really work. This happens even before we check signatures: we did so
+     * before when we stored this to disk. This does mean an attacker who can
+     * write to the datadir can make us not start: such an attacker could
+     * already harm us by replacing our guards, which would be worse. */
+    checked_protocols_already = 1;
+    handle_missing_protocol_warning(c, options);
+  }
+
+  if ((int)c->flavor != flav) {
+    /* This wasn't the flavor we thought we were getting. */
+    if (require_flavor) {
+      log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)",
+               networkstatus_get_flavor_name(c->flavor), flavor);
+      goto done;
+    }
+    flav = c->flavor;
+    flavor = networkstatus_get_flavor_name(flav);
+  }
+
+  if (flav != usable_consensus_flavor() &&
+      !we_want_to_fetch_flavor(options, flav)) {
+    /* This consensus is totally boring to us: we won't use it, we didn't want
+     * it, and we won't serve it.  Drop it. */
+    goto done;
+  }
+
+  if (from_cache && !accept_obsolete &&
+      c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) {
+    log_info(LD_DIR, "Loaded an expired consensus. Discarding.");
+    goto done;
+  }
+
+  if (!strcmp(flavor, "ns")) {
+    consensus_fname = get_cachedir_fname("cached-consensus");
+    unverified_fname = get_cachedir_fname("unverified-consensus");
+    if (current_ns_consensus) {
+      current_digests = &current_ns_consensus->digests;
+      current_valid_after = current_ns_consensus->valid_after;
+    }
+  } else if (!strcmp(flavor, "microdesc")) {
+    consensus_fname = get_cachedir_fname("cached-microdesc-consensus");
+    unverified_fname = get_cachedir_fname("unverified-microdesc-consensus");
+    if (current_md_consensus) {
+      current_digests = &current_md_consensus->digests;
+      current_valid_after = current_md_consensus->valid_after;
+    }
+  } else {
+    tor_assert_nonfatal_unreached();
+    result = -2;
+    goto done;
+  }
+
+  if (current_digests &&
+      tor_memeq(&c->digests, current_digests, sizeof(c->digests))) {
+    /* We already have this one. That's a failure. */
+    log_info(LD_DIR, "Got a %s consensus we already have", flavor);
+    goto done;
+  }
+
+  if (current_valid_after && c->valid_after <= current_valid_after) {
+    /* We have a newer one.  There's no point in accepting this one,
+     * even if it's great. */
+    log_info(LD_DIR, "Got a %s consensus at least as old as the one we have",
+             flavor);
+    goto done;
+  }
+
+  /* Make sure it's signed enough. */
+  if ((r=networkstatus_check_consensus_signature(c, 1))<0) {
+    if (r == -1) {
+      /* Okay, so it _might_ be signed enough if we get more certificates. */
+      if (!was_waiting_for_certs) {
+        log_info(LD_DIR,
+                 "Not enough certificates to check networkstatus consensus");
+      }
+      if (!current_valid_after ||
+          c->valid_after > current_valid_after) {
+        waiting = &consensus_waiting_for_certs[flav];
+        networkstatus_vote_free(waiting->consensus);
+        tor_free(waiting->body);
+        waiting->consensus = c;
+        free_consensus = 0;
+        waiting->body = tor_strdup(consensus);
+        waiting->set_at = now;
+        waiting->dl_failed = 0;
+        if (!from_cache) {
+          write_str_to_file(unverified_fname, consensus, 0);
+        }
+        if (dl_certs)
+          authority_certs_fetch_missing(c, now, source_dir);
+        /* This case is not a success or a failure until we get the certs
+         * or fail to get the certs. */
+        result = 0;
+      } else {
+        /* Even if we had enough signatures, we'd never use this as the
+         * latest consensus. */
+        if (was_waiting_for_certs && from_cache)
+          if (unlink(unverified_fname) != 0) {
+            log_warn(LD_FS,
+                     "Failed to unlink %s: %s",
+                     unverified_fname, strerror(errno));
+          }
+      }
+      goto done;
+    } else {
+      /* This can never be signed enough:  Kill it. */
+      if (!was_waiting_for_certs) {
+        log_warn(LD_DIR, "Not enough good signatures on networkstatus "
+                 "consensus");
+        result = -2;
+      }
+      if (was_waiting_for_certs && (r < -1) && from_cache) {
+        if (unlink(unverified_fname) != 0) {
+            log_warn(LD_FS,
+                     "Failed to unlink %s: %s",
+                     unverified_fname, strerror(errno));
+        }
+      }
+      goto done;
+    }
+  }
+
+  /* Signatures from the consensus are verified */
+  if (from_cache && was_waiting_for_certs) {
+    /* We check if the consensus is loaded from disk cache and that it
+     * it is an unverified consensus. If it is unverified, rename it to
+     * cached-*-consensus since it has been verified. */
+    log_info(LD_DIR, "Unverified consensus signatures verified.");
+    tor_rename(unverified_fname, consensus_fname);
+  }
+
+  if (!from_cache && flav == usable_consensus_flavor())
+    control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED");
+
+  if (!checked_protocols_already) {
+    handle_missing_protocol_warning(c, options);
+  }
+
+  /* Are we missing any certificates at all? */
+  if (r != 1 && dl_certs)
+    authority_certs_fetch_missing(c, now, source_dir);
+
+  const int is_usable_flavor = flav == usable_consensus_flavor();
+
+  /* Before we switch to the new consensus, notify that we are about to change
+   * it using the old consensus and the new one. */
+  if (is_usable_flavor) {
+    notify_before_networkstatus_changes(networkstatus_get_latest_consensus(),
+                                        c);
+  }
+  if (flav == FLAV_NS) {
+    if (current_ns_consensus) {
+      networkstatus_copy_old_consensus_info(c, current_ns_consensus);
+      networkstatus_vote_free(current_ns_consensus);
+      /* Defensive programming : we should set current_ns_consensus very soon
+       * but we're about to call some stuff in the meantime, and leaving this
+       * dangling pointer around has proven to be trouble. */
+      current_ns_consensus = NULL;
+    }
+    current_ns_consensus = c;
+    free_consensus = 0; /* avoid free */
+  } else if (flav == FLAV_MICRODESC) {
+    if (current_md_consensus) {
+      networkstatus_copy_old_consensus_info(c, current_md_consensus);
+      networkstatus_vote_free(current_md_consensus);
+      /* more defensive programming */
+      current_md_consensus = NULL;
+    }
+    current_md_consensus = c;
+    free_consensus = 0; /* avoid free */
+  }
+
+  waiting = &consensus_waiting_for_certs[flav];
+  if (waiting->consensus &&
+      waiting->consensus->valid_after <= c->valid_after) {
+    networkstatus_vote_free(waiting->consensus);
+    waiting->consensus = NULL;
+    if (consensus != waiting->body)
+      tor_free(waiting->body);
+    else
+      waiting->body = NULL;
+    waiting->set_at = 0;
+    waiting->dl_failed = 0;
+    if (unlink(unverified_fname) != 0) {
+      log_warn(LD_FS,
+               "Failed to unlink %s: %s",
+               unverified_fname, strerror(errno));
+    }
+  }
+
+  if (is_usable_flavor) {
+    /* Notify that we just changed the consensus so the current global value
+     * can be looked at. */
+    notify_after_networkstatus_changes();
+
+    /* The "current" consensus has just been set and it is a usable flavor so
+     * the first thing we need to do is recalculate the voting schedule static
+     * object so we can use the timings in there needed by some subsystems
+     * such as hidden service and shared random. */
+    voting_schedule_recalculate_timing(options, now);
+    reschedule_dirvote(options);
+
+    nodelist_set_consensus(c);
+
+    /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/
+    update_consensus_networkstatus_fetch_time(now);
+
+    /* Change the cell EWMA settings */
+    cmux_ewma_set_options(options, c);
+
+    /* XXXX this call might be unnecessary here: can changing the
+     * current consensus really alter our view of any OR's rate limits? */
+    connection_or_update_token_buckets(get_connection_array(), options);
+
+    circuit_build_times_new_consensus_params(
+                               get_circuit_build_times_mutable(), c);
+    channelpadding_new_consensus_params(c);
+  }
+
+  /* Reset the failure count only if this consensus is actually valid. */
+  if (c->valid_after <= now && now <= c->valid_until) {
+    download_status_reset(&consensus_dl_status[flav]);
+  } else {
+    if (!from_cache)
+      download_status_failed(&consensus_dl_status[flav], 0);
+  }
+
+  if (we_want_to_fetch_flavor(options, flav)) {
+    dirserv_set_cached_consensus_networkstatus(consensus,
+                                               flavor,
+                                               &c->digests,
+                                               c->digest_sha3_as_signed,
+                                               c->valid_after);
+    if (dir_server_mode(get_options())) {
+      consdiffmgr_add_consensus(consensus, c);
+    }
+  }
+
+  if (!from_cache) {
+    write_str_to_file(consensus_fname, consensus, 0);
+  }
+
+  warn_early_consensus(c, flavor, now);
+
+  /* We got a new consesus. Reset our md fetch fail cache */
+  microdesc_reset_outdated_dirservers_list();
+
+  router_dir_info_changed();
+
+  result = 0;
+ done:
+  if (free_consensus)
+    networkstatus_vote_free(c);
+  tor_free(consensus_fname);
+  tor_free(unverified_fname);
+  return result;
+}
+
+/** Called when we have gotten more certificates: see whether we can
+ * now verify a pending consensus.
+ *
+ * If source_dir is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved certificates from, so try it first to
+ * fetch any missing certificates.
+ */
+void
+networkstatus_note_certs_arrived(const char *source_dir)
+{
+  int i;
+  for (i=0; i<N_CONSENSUS_FLAVORS; ++i) {
+    const char *flavor_name = networkstatus_get_flavor_name(i);
+    consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+    if (!waiting->consensus)
+      continue;
+    if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) {
+      char *waiting_body = waiting->body;
+      if (!networkstatus_set_current_consensus(
+                                 waiting_body,
+                                 flavor_name,
+                                 NSSET_WAS_WAITING_FOR_CERTS,
+                                 source_dir)) {
+        tor_free(waiting_body);
+      }
+    }
+  }
+}
+
+/** If the network-status list has changed since the last time we called this
+ * function, update the status of every routerinfo from the network-status
+ * list. If <b>dir_version</b> is 2, it's a v2 networkstatus that changed.
+ * If <b>dir_version</b> is 3, it's a v3 consensus that changed.
+ */
+void
+routers_update_all_from_networkstatus(time_t now, int dir_version)
+{
+  routerlist_t *rl = router_get_routerlist();
+  networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,
+                                                                     FLAV_NS);
+
+  if (!consensus || dir_version < 3) /* nothing more we should do */
+    return;
+
+  /* calls router_dir_info_changed() when it's done -- more routers
+   * might be up or down now, which might affect whether there's enough
+   * directory info. */
+  routers_update_status_from_consensus_networkstatus(rl->routers, 0);
+
+  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri,
+                    ri->cache_info.routerlist_index = ri_sl_idx);
+  if (rl->old_routers)
+    signed_descs_update_status_from_consensus_networkstatus(rl->old_routers);
+
+  if (!have_warned_about_old_version) {
+    int is_server = server_mode(get_options());
+    version_status_t status;
+    const char *recommended = is_server ?
+      consensus->server_versions : consensus->client_versions;
+    status = tor_version_is_obsolete(VERSION, recommended);
+
+    if (status == VS_RECOMMENDED) {
+      log_info(LD_GENERAL, "The directory authorities say my version is ok.");
+    } else if (status == VS_EMPTY) {
+      log_info(LD_GENERAL,
+               "The directory authorities don't recommend any versions.");
+    } else if (status == VS_NEW || status == VS_NEW_IN_SERIES) {
+      if (!have_warned_about_new_version) {
+        log_notice(LD_GENERAL, "This version of Tor (%s) is newer than any "
+                   "recommended version%s, according to the directory "
+                   "authorities. Recommended versions are: %s",
+                   VERSION,
+                   status == VS_NEW_IN_SERIES ? " in its series" : "",
+                   recommended);
+        have_warned_about_new_version = 1;
+        control_event_general_status(LOG_WARN, "DANGEROUS_VERSION "
+                                     "CURRENT=%s REASON=%s RECOMMENDED=\"%s\"",
+                                     VERSION, "NEW", recommended);
+      }
+    } else {
+      log_warn(LD_GENERAL, "Please upgrade! "
+               "This version of Tor (%s) is %s, according to the directory "
+               "authorities. Recommended versions are: %s",
+               VERSION,
+               status == VS_OLD ? "obsolete" : "not recommended",
+               recommended);
+      have_warned_about_old_version = 1;
+      control_event_general_status(LOG_WARN, "DANGEROUS_VERSION "
+           "CURRENT=%s REASON=%s RECOMMENDED=\"%s\"",
+           VERSION, status == VS_OLD ? "OBSOLETE" : "UNRECOMMENDED",
+           recommended);
+    }
+  }
+}
+
+/** Given a list <b>routers</b> of routerinfo_t *, update each status field
+ * according to our current consensus networkstatus.  May re-order
+ * <b>routers</b>. */
+void
+routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
+                                                   int reset_failures)
+{
+  const or_options_t *options = get_options();
+  int authdir = authdir_mode_v3(options);
+  networkstatus_t *ns = networkstatus_get_latest_consensus();
+  if (!ns || !smartlist_len(ns->routerstatus_list))
+    return;
+
+  routers_sort_by_identity(routers);
+
+  SMARTLIST_FOREACH_JOIN(ns->routerstatus_list, routerstatus_t *, rs,
+                         routers, routerinfo_t *, router,
+                         tor_memcmp(rs->identity_digest,
+                               router->cache_info.identity_digest, DIGEST_LEN),
+  {
+  }) {
+    /* Is it the same descriptor, or only the same identity? */
+    if (tor_memeq(router->cache_info.signed_descriptor_digest,
+                rs->descriptor_digest, DIGEST_LEN)) {
+      if (ns->valid_until > router->cache_info.last_listed_as_valid_until)
+        router->cache_info.last_listed_as_valid_until = ns->valid_until;
+    }
+
+    if (authdir) {
+      /* If we _are_ an authority, we should check whether this router
+       * is one that will cause us to need a reachability test. */
+      routerinfo_t *old_router =
+        router_get_mutable_by_digest(router->cache_info.identity_digest);
+      if (old_router != router) {
+        router->needs_retest_if_added =
+          dirserv_should_launch_reachability_test(router, old_router);
+      }
+    }
+    if (reset_failures) {
+      download_status_reset(&rs->dl_status);
+    }
+  } SMARTLIST_FOREACH_JOIN_END(rs, router);
+
+  router_dir_info_changed();
+}
+
+/** Given a list of signed_descriptor_t, update their fields (mainly, when
+ * they were last listed) from the most recent consensus. */
+void
+signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
+{
+  networkstatus_t *ns = current_ns_consensus;
+  if (!ns)
+    return;
+
+  if (!ns->desc_digest_map) {
+    char dummy[DIGEST_LEN];
+    /* instantiates the digest map. */
+    memset(dummy, 0, sizeof(dummy));
+    router_get_consensus_status_by_descriptor_digest(ns, dummy);
+  }
+  SMARTLIST_FOREACH(descs, signed_descriptor_t *, d,
+  {
+    const routerstatus_t *rs = digestmap_get(ns->desc_digest_map,
+                                       d->signed_descriptor_digest);
+    if (rs) {
+      if (ns->valid_until > d->last_listed_as_valid_until)
+        d->last_listed_as_valid_until = ns->valid_until;
+    }
+  });
+}
+
+/** Generate networkstatus lines for a single routerstatus_t object, and
+ * return the result in a newly allocated string.  Used only by controller
+ * interface (for now.) */
+char *
+networkstatus_getinfo_helper_single(const routerstatus_t *rs)
+{
+  return routerstatus_format_entry(rs, NULL, NULL, NS_CONTROL_PORT,
+                                   ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
+                                   NULL);
+}
+
+/** Alloc and return a string describing routerstatuses for the most
+ * recent info of each router we know about that is of purpose
+ * <b>purpose_string</b>. Return NULL if unrecognized purpose.
+ *
+ * Right now this function is oriented toward listing bridges (you
+ * shouldn't use this for general-purpose routers, since those
+ * should be listed from the consensus, not from the routers list). */
+char *
+networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
+{
+  const time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  char *answer;
+  routerlist_t *rl = router_get_routerlist();
+  smartlist_t *statuses;
+  const uint8_t purpose = router_purpose_from_string(purpose_string);
+  routerstatus_t rs;
+  const int bridge_auth = authdir_mode_bridge(get_options());
+
+  if (purpose == ROUTER_PURPOSE_UNKNOWN) {
+    log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.",
+             purpose_string);
+    return NULL;
+  }
+
+  statuses = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) {
+    node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+    if (!node)
+      continue;
+    if (ri->cache_info.published_on < cutoff)
+      continue;
+    if (ri->purpose != purpose)
+      continue;
+    /* TODO: modifying the running flag in a getinfo is a bad idea */
+    if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE)
+      dirserv_set_router_is_running(ri, now);
+    /* then generate and write out status lines for each of them */
+    set_routerstatus_from_routerinfo(&rs, node, ri, now, 0);
+    smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs));
+  } SMARTLIST_FOREACH_END(ri);
+
+  answer = smartlist_join_strings(statuses, "", 0, NULL);
+  SMARTLIST_FOREACH(statuses, char *, cp, tor_free(cp));
+  smartlist_free(statuses);
+  return answer;
+}
+
+/** Write out router status entries for all our bridge descriptors. */
+void
+networkstatus_dump_bridge_status_to_file(time_t now)
+{
+  char *status = networkstatus_getinfo_by_purpose("bridge", now);
+  char *fname = NULL;
+  char *thresholds = NULL;
+  char *published_thresholds_and_status = NULL;
+  char published[ISO_TIME_LEN+1];
+  const routerinfo_t *me = router_get_my_routerinfo();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *fingerprint_line = NULL;
+
+  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
+                                      fingerprint, 0) >= 0) {
+    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
+  } else {
+    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
+  }
+  format_iso_time(published, now);
+  dirserv_compute_bridge_flag_thresholds();
+  thresholds = dirserv_get_flag_thresholds_line();
+  tor_asprintf(&published_thresholds_and_status,
+               "published %s\nflag-thresholds %s\n%s%s",
+               published, thresholds, fingerprint_line ? fingerprint_line : "",
+               status);
+  fname = get_datadir_fname("networkstatus-bridges");
+  write_str_to_file(fname,published_thresholds_and_status,0);
+  tor_free(thresholds);
+  tor_free(published_thresholds_and_status);
+  tor_free(fname);
+  tor_free(status);
+  tor_free(fingerprint_line);
+}
+
+/* DOCDOC get_net_param_from_list */
+static int32_t
+get_net_param_from_list(smartlist_t *net_params, const char *param_name,
+                        int32_t default_val, int32_t min_val, int32_t max_val)
+{
+  int32_t res = default_val;
+  size_t name_len = strlen(param_name);
+
+  tor_assert(max_val > min_val);
+  tor_assert(min_val <= default_val);
+  tor_assert(max_val >= default_val);
+
+  SMARTLIST_FOREACH_BEGIN(net_params, const char *, p) {
+    if (!strcmpstart(p, param_name) && p[name_len] == '=') {
+      int ok=0;
+      long v = tor_parse_long(p+name_len+1, 10, INT32_MIN,
+                              INT32_MAX, &ok, NULL);
+      if (ok) {
+        res = (int32_t) v;
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(p);
+
+  if (res < min_val) {
+    log_warn(LD_DIR, "Consensus parameter %s is too small. Got %d, raising to "
+             "%d.", param_name, res, min_val);
+    res = min_val;
+  } else if (res > max_val) {
+    log_warn(LD_DIR, "Consensus parameter %s is too large. Got %d, capping to "
+             "%d.", param_name, res, max_val);
+    res = max_val;
+  }
+
+  return res;
+}
+
+/** Return the value of a integer parameter from the networkstatus <b>ns</b>
+ * whose name is <b>param_name</b>.  If <b>ns</b> is NULL, try loading the
+ * latest consensus ourselves. Return <b>default_val</b> if no latest
+ * consensus, or if it has no parameter called <b>param_name</b>.
+ * Make sure the value parsed from the consensus is at least
+ * <b>min_val</b> and at most <b>max_val</b> and raise/cap the parsed value
+ * if necessary. */
+MOCK_IMPL(int32_t,
+networkstatus_get_param, (const networkstatus_t *ns, const char *param_name,
+                        int32_t default_val, int32_t min_val, int32_t max_val))
+{
+  if (!ns) /* if they pass in null, go find it ourselves */
+    ns = networkstatus_get_latest_consensus();
+
+  if (!ns || !ns->net_params)
+    return default_val;
+
+  return get_net_param_from_list(ns->net_params, param_name,
+                                 default_val, min_val, max_val);
+}
+
+/**
+ * As networkstatus_get_param(), but check torrc_value before checking the
+ * consensus. If torrc_value is in-range, then return it instead of the
+ * value from the consensus.
+ */
+int32_t
+networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                    int32_t torrc_value,
+                                    const char *param_name,
+                                    int32_t default_val,
+                                    int32_t min_val, int32_t max_val)
+{
+  if (torrc_value >= min_val && torrc_value <= max_val)
+    return torrc_value;
+  else
+    return networkstatus_get_param(
+                         ns, param_name, default_val, min_val, max_val);
+}
+
+/**
+ * Retrieve the consensus parameter that governs the
+ * fixed-point precision of our network balancing 'bandwidth-weights'
+ * (which are themselves integer consensus values). We divide them
+ * by this value and ensure they never exceed this value.
+ */
+int
+networkstatus_get_weight_scale_param(networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "bwweightscale",
+                                 BW_WEIGHT_SCALE,
+                                 BW_MIN_WEIGHT_SCALE,
+                                 BW_MAX_WEIGHT_SCALE);
+}
+
+/** Return the value of a integer bw weight parameter from the networkstatus
+ * <b>ns</b> whose name is <b>weight_name</b>.  If <b>ns</b> is NULL, try
+ * loading the latest consensus ourselves. Return <b>default_val</b> if no
+ * latest consensus, or if it has no parameter called <b>weight_name</b>. */
+int32_t
+networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight_name,
+                            int32_t default_val)
+{
+  int32_t param;
+  int max;
+  if (!ns) /* if they pass in null, go find it ourselves */
+    ns = networkstatus_get_latest_consensus();
+
+  if (!ns || !ns->weight_params)
+    return default_val;
+
+  max = networkstatus_get_weight_scale_param(ns);
+  param = get_net_param_from_list(ns->weight_params, weight_name,
+                                  default_val, -1,
+                                  BW_MAX_WEIGHT_SCALE);
+  if (param > max) {
+    log_warn(LD_DIR, "Value of consensus weight %s was too large, capping "
+             "to %d", weight_name, max);
+    param = max;
+  }
+  return param;
+}
+
+/** Return the name of the consensus flavor <b>flav</b> as used to identify
+ * the flavor in directory documents. */
+const char *
+networkstatus_get_flavor_name(consensus_flavor_t flav)
+{
+  switch (flav) {
+    case FLAV_NS:
+      return "ns";
+    case FLAV_MICRODESC:
+      return "microdesc";
+    default:
+      tor_fragile_assert();
+      return "??";
+  }
+}
+
+/** Return the consensus_flavor_t value for the flavor called <b>flavname</b>,
+ * or -1 if the flavor is not recognized. */
+int
+networkstatus_parse_flavor_name(const char *flavname)
+{
+  if (!strcmp(flavname, "ns"))
+    return FLAV_NS;
+  else if (!strcmp(flavname, "microdesc"))
+    return FLAV_MICRODESC;
+  else
+    return -1;
+}
+
+/** Return 0 if this routerstatus is obsolete, too new, isn't
+ * running, or otherwise not a descriptor that we would make any
+ * use of even if we had it. Else return 1. */
+int
+client_would_use_router(const routerstatus_t *rs, time_t now)
+{
+  if (!rs->is_flagged_running) {
+    /* If we had this router descriptor, we wouldn't even bother using it.
+     * (Fetching and storing depends on by we_want_to_fetch_flavor().) */
+    return 0;
+  }
+  if (rs->published_on + OLD_ROUTER_DESC_MAX_AGE < now) {
+    /* We'd drop it immediately for being too old. */
+    return 0;
+  }
+  if (!routerstatus_version_supports_extend2_cells(rs, 1)) {
+    /* We'd ignore it because it doesn't support EXTEND2 cells.
+     * If we don't know the version, download the descriptor so we can
+     * check if it supports EXTEND2 cells and ntor. */
+    return 0;
+  }
+  return 1;
+}
+
+/** If <b>question</b> is a string beginning with "ns/" in a format the
+ * control interface expects for a GETINFO question, set *<b>answer</b> to a
+ * newly-allocated string containing networkstatus lines for the appropriate
+ * ORs.  Return 0 on success, -1 on unrecognized question format. */
+int
+getinfo_helper_networkstatus(control_connection_t *conn,
+                             const char *question, char **answer,
+                             const char **errmsg)
+{
+  const routerstatus_t *status;
+  (void) conn;
+
+  if (!networkstatus_get_latest_consensus()) {
+    *answer = tor_strdup("");
+    return 0;
+  }
+
+  if (!strcmp(question, "ns/all")) {
+    smartlist_t *statuses = smartlist_new();
+    SMARTLIST_FOREACH(networkstatus_get_latest_consensus()->routerstatus_list,
+                      const routerstatus_t *, rs,
+      {
+        smartlist_add(statuses, networkstatus_getinfo_helper_single(rs));
+      });
+    *answer = smartlist_join_strings(statuses, "", 0, NULL);
+    SMARTLIST_FOREACH(statuses, char *, cp, tor_free(cp));
+    smartlist_free(statuses);
+    return 0;
+  } else if (!strcmpstart(question, "ns/id/")) {
+    char d[DIGEST_LEN];
+    const char *q = question + 6;
+    if (*q == '$')
+      ++q;
+
+    if (base16_decode(d, DIGEST_LEN, q, strlen(q)) != DIGEST_LEN) {
+      *errmsg = "Data not decodeable as hex";
+      return -1;
+    }
+    status = router_get_consensus_status_by_id(d);
+  } else if (!strcmpstart(question, "ns/name/")) {
+    const node_t *n = node_get_by_nickname(question+8, 0);
+    status = n ? n->rs : NULL;
+  } else if (!strcmpstart(question, "ns/purpose/")) {
+    *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL));
+    return *answer ? 0 : -1;
+  } else if (!strcmp(question, "consensus/packages")) {
+    const networkstatus_t *ns = networkstatus_get_latest_consensus();
+    if (ns && ns->package_lines)
+      *answer = smartlist_join_strings(ns->package_lines, "\n", 0, NULL);
+    else
+      *errmsg = "No consensus available";
+    return *answer ? 0 : -1;
+  } else if (!strcmp(question, "consensus/valid-after") ||
+             !strcmp(question, "consensus/fresh-until") ||
+             !strcmp(question, "consensus/valid-until")) {
+    const networkstatus_t *ns = networkstatus_get_latest_consensus();
+    if (ns) {
+      time_t t;
+      if (!strcmp(question, "consensus/valid-after"))
+        t = ns->valid_after;
+      else if (!strcmp(question, "consensus/fresh-until"))
+        t = ns->fresh_until;
+      else
+        t = ns->valid_until;
+
+      char tbuf[ISO_TIME_LEN+1];
+      format_iso_time(tbuf, t);
+      *answer = tor_strdup(tbuf);
+    } else {
+      *errmsg = "No consensus available";
+    }
+    return *answer ? 0 : -1;
+  } else {
+    return 0;
+  }
+
+  if (status)
+    *answer = networkstatus_getinfo_helper_single(status);
+  return 0;
+}
+
+/** Check whether the networkstatus <b>ns</b> lists any protocol
+ * versions as "required" or "recommended" that we do not support.  If
+ * so, set *<b>warning_out</b> to a newly allocated string describing
+ * the problem.
+ *
+ * Return 1 if we should exit, 0 if we should not. */
+int
+networkstatus_check_required_protocols(const networkstatus_t *ns,
+                                       int client_mode,
+                                       char **warning_out)
+{
+  const char *func = client_mode ? "client" : "relay";
+  const char *required, *recommended;
+  char *missing = NULL;
+
+  tor_assert(warning_out);
+
+  if (client_mode) {
+    required = ns->required_client_protocols;
+    recommended = ns->recommended_client_protocols;
+  } else {
+    required = ns->required_relay_protocols;
+    recommended = ns->recommended_relay_protocols;
+  }
+
+  if (!protover_all_supported(required, &missing)) {
+    tor_asprintf(warning_out, "At least one protocol listed as required in "
+                 "the consensus is not supported by this version of Tor. "
+                 "You should upgrade. This version of Tor will not work as a "
+                 "%s on the Tor network. The missing protocols are: %s",
+                 func, missing);
+    tor_free(missing);
+    return 1;
+  }
+
+  if (! protover_all_supported(recommended, &missing)) {
+    tor_asprintf(warning_out, "At least one protocol listed as recommended in "
+                 "the consensus is not supported by this version of Tor. "
+                 "You should upgrade. This version of Tor will eventually "
+                 "stop working as a %s on the Tor network. The missing "
+                 "protocols are: %s",
+                 func, missing);
+    tor_free(missing);
+  }
+
+  tor_assert_nonfatal(missing == NULL);
+
+  return 0;
+}
+
+/** Release all storage held in <b>s</b>. */
+void
+ns_detached_signatures_free_(ns_detached_signatures_t *s)
+{
+  if (!s)
+    return;
+  if (s->signatures) {
+    STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) {
+      SMARTLIST_FOREACH(sigs, document_signature_t *, sig,
+                        document_signature_free(sig));
+      smartlist_free(sigs);
+    } STRMAP_FOREACH_END;
+    strmap_free(s->signatures, NULL);
+    strmap_free(s->digests, tor_free_);
+  }
+
+  tor_free(s);
+}
+
+/** Free all storage held locally in this module. */
+void
+networkstatus_free_all(void)
+{
+  int i;
+  networkstatus_vote_free(current_ns_consensus);
+  networkstatus_vote_free(current_md_consensus);
+  current_md_consensus = current_ns_consensus = NULL;
+
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
+    consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+    if (waiting->consensus) {
+      networkstatus_vote_free(waiting->consensus);
+      waiting->consensus = NULL;
+    }
+    tor_free(waiting->body);
+  }
+}
diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h
new file mode 100644
index 0000000000..cc6badf0b2
--- /dev/null
+++ b/src/feature/nodelist/networkstatus.h
@@ -0,0 +1,162 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file networkstatus.h
+ * \brief Header file for networkstatus.c.
+ **/
+
+#ifndef TOR_NETWORKSTATUS_H
+#define TOR_NETWORKSTATUS_H
+
+#include "lib/testsupport/testsupport.h"
+
+void networkstatus_reset_warnings(void);
+void networkstatus_reset_download_failures(void);
+char *networkstatus_read_cached_consensus(const char *flavorname);
+int router_reload_consensus_networkstatus(void);
+void routerstatus_free_(routerstatus_t *rs);
+#define routerstatus_free(rs) \
+  FREE_AND_NULL(routerstatus_t, routerstatus_free_, (rs))
+void networkstatus_vote_free_(networkstatus_t *ns);
+#define networkstatus_vote_free(ns) \
+  FREE_AND_NULL(networkstatus_t, networkstatus_vote_free_, (ns))
+void ns_detached_signatures_free_(ns_detached_signatures_t *s);
+#define ns_detached_signatures_free(s) \
+  FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s))
+networkstatus_voter_info_t *networkstatus_get_voter_by_id(
+                                       networkstatus_t *vote,
+                                       const char *identity);
+document_signature_t *networkstatus_get_voter_sig_by_alg(
+                                    const networkstatus_voter_info_t *voter,
+                                    digest_algorithm_t alg);
+
+int networkstatus_check_consensus_signature(networkstatus_t *consensus,
+                                            int warn);
+int networkstatus_check_document_signature(const networkstatus_t *consensus,
+                                           document_signature_t *sig,
+                                           const authority_cert_t *cert);
+int compare_digest_to_routerstatus_entry(const void *_key,
+                                         const void **_member);
+int compare_digest_to_vote_routerstatus_entry(const void *_key,
+                                              const void **_member);
+const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns,
+                                              const char *digest);
+routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns,
+                                              const char *digest);
+int networkstatus_vote_find_entry_idx(networkstatus_t *ns,
+                                      const char *digest, int *found_out);
+
+MOCK_DECL(download_status_t *,
+  networkstatus_get_dl_status_by_flavor,
+  (consensus_flavor_t flavor));
+MOCK_DECL(download_status_t *,
+  networkstatus_get_dl_status_by_flavor_bootstrap,
+  (consensus_flavor_t flavor));
+MOCK_DECL(download_status_t *,
+  networkstatus_get_dl_status_by_flavor_running,
+  (consensus_flavor_t flavor));
+
+MOCK_DECL(smartlist_t *, router_get_descriptor_digests, (void));
+MOCK_DECL(download_status_t *,router_get_dl_status_by_descriptor_digest,
+          (const char *d));
+
+const routerstatus_t *router_get_consensus_status_by_id(const char *digest);
+routerstatus_t *router_get_mutable_consensus_status_by_id(
+                                   const char *digest);
+const routerstatus_t *router_get_consensus_status_by_descriptor_digest(
+                                   networkstatus_t *consensus,
+                                   const char *digest);
+MOCK_DECL(routerstatus_t *,
+          router_get_mutable_consensus_status_by_descriptor_digest,
+          (networkstatus_t *consensus, const char *digest));
+int we_want_to_fetch_flavor(const or_options_t *options, int flavor);
+int we_want_to_fetch_unknown_auth_certs(const or_options_t *options);
+void networkstatus_consensus_download_failed(int status_code,
+                                             const char *flavname);
+void update_consensus_networkstatus_fetch_time(time_t now);
+int should_delay_dir_fetches(const or_options_t *options,const char **msg_out);
+void update_networkstatus_downloads(time_t now);
+void update_certificate_downloads(time_t now);
+int consensus_is_waiting_for_certs(void);
+int client_would_use_router(const routerstatus_t *rs, time_t now);
+MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus,(void));
+MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor,
+          (consensus_flavor_t f));
+MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
+int networkstatus_is_live(const networkstatus_t *ns, time_t now);
+int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                            time_t now);
+int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                                 time_t now);
+networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now,
+                                                             int flavor);
+MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now));
+int networkstatus_consensus_can_use_multiple_directories(
+                                                const or_options_t *options);
+MOCK_DECL(int, networkstatus_consensus_can_use_extra_fallbacks,(
+                                                const or_options_t *options));
+int networkstatus_consensus_is_already_downloading(const char *resource);
+int networkstatus_consensus_has_ipv6(const or_options_t* options);
+
+#define NSSET_FROM_CACHE 1
+#define NSSET_WAS_WAITING_FOR_CERTS 2
+#define NSSET_DONT_DOWNLOAD_CERTS 4
+#define NSSET_ACCEPT_OBSOLETE 8
+#define NSSET_REQUIRE_FLAVOR 16
+int networkstatus_set_current_consensus(const char *consensus,
+                                        const char *flavor,
+                                        unsigned flags,
+                                        const char *source_dir);
+void networkstatus_note_certs_arrived(const char *source_dir);
+void routers_update_all_from_networkstatus(time_t now, int dir_version);
+void routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
+                                                        int reset_failures);
+void signed_descs_update_status_from_consensus_networkstatus(
+                                                         smartlist_t *descs);
+
+char *networkstatus_getinfo_helper_single(const routerstatus_t *rs);
+char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now);
+void networkstatus_dump_bridge_status_to_file(time_t now);
+MOCK_DECL(int32_t, networkstatus_get_param,
+          (const networkstatus_t *ns, const char *param_name,
+           int32_t default_val, int32_t min_val, int32_t max_val));
+int32_t networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                            int32_t torrc_value,
+                                            const char *param_name,
+                                            int32_t default_val,
+                                            int32_t min_val, int32_t max_val);
+int getinfo_helper_networkstatus(control_connection_t *conn,
+                                 const char *question, char **answer,
+                                 const char **errmsg);
+int32_t networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight,
+                                    int32_t default_val);
+const char *networkstatus_get_flavor_name(consensus_flavor_t flav);
+int networkstatus_parse_flavor_name(const char *flavname);
+void document_signature_free_(document_signature_t *sig);
+#define document_signature_free(sig) \
+  FREE_AND_NULL(document_signature_t, document_signature_free_, (sig))
+document_signature_t *document_signature_dup(const document_signature_t *sig);
+void networkstatus_free_all(void);
+int networkstatus_get_weight_scale_param(networkstatus_t *ns);
+
+void vote_routerstatus_free_(vote_routerstatus_t *rs);
+#define vote_routerstatus_free(rs) \
+  FREE_AND_NULL(vote_routerstatus_t, vote_routerstatus_free_, (rs))
+
+#ifdef NETWORKSTATUS_PRIVATE
+#ifdef TOR_UNIT_TESTS
+STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
+                                                const char *flavor);
+STATIC void warn_early_consensus(const networkstatus_t *c, const char *flavor,
+                                 time_t now);
+extern networkstatus_t *current_ns_consensus;
+extern networkstatus_t *current_md_consensus;
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(NETWORKSTATUS_PRIVATE) */
+
+#endif /* !defined(TOR_NETWORKSTATUS_H) */
+
diff --git a/src/feature/nodelist/networkstatus_sr_info_st.h b/src/feature/nodelist/networkstatus_sr_info_st.h
new file mode 100644
index 0000000000..6c937a75f5
--- /dev/null
+++ b/src/feature/nodelist/networkstatus_sr_info_st.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NETWORKSTATUS_SR_INFO_ST_H
+#define NETWORKSTATUS_SR_INFO_ST_H
+
+struct networkstatus_sr_info_t {
+  /* Indicate if the dirauth partitipates in the SR protocol with its vote.
+   * This is tied to the SR flag in the vote. */
+  unsigned int participate:1;
+  /* Both vote and consensus: Current and previous SRV. If list is empty,
+   * this means none were found in either the consensus or vote. */
+  struct sr_srv_t *previous_srv;
+  struct sr_srv_t *current_srv;
+  /* Vote only: List of commitments. */
+  smartlist_t *commits;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h
new file mode 100644
index 0000000000..4a193ad149
--- /dev/null
+++ b/src/feature/nodelist/networkstatus_st.h
@@ -0,0 +1,101 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NETWORKSTATUS_ST_H
+#define NETWORKSTATUS_ST_H
+
+#include "or/networkstatus_sr_info_st.h"
+
+/** Enumerates the possible seriousness values of a networkstatus document. */
+typedef enum networkstatus_type_t {
+  NS_TYPE_VOTE,
+  NS_TYPE_CONSENSUS,
+  NS_TYPE_OPINION,
+} networkstatus_type_t;
+
+/** A common structure to hold a v3 network status vote, or a v3 network
+ * status consensus. */
+struct networkstatus_t {
+  networkstatus_type_t type; /**< Vote, consensus, or opinion? */
+  consensus_flavor_t flavor; /**< If a consensus, what kind? */
+  unsigned int has_measured_bws : 1;/**< True iff this networkstatus contains
+                                     * measured= bandwidth values. */
+
+  time_t published; /**< Vote only: Time when vote was written. */
+  time_t valid_after; /**< Time after which this vote or consensus applies. */
+  time_t fresh_until; /**< Time before which this is the most recent vote or
+                       * consensus. */
+  time_t valid_until; /**< Time after which this vote or consensus should not
+                       * be used. */
+
+  /** Consensus only: what method was used to produce this consensus? */
+  int consensus_method;
+  /** Vote only: what methods is this voter willing to use? */
+  smartlist_t *supported_methods;
+
+  /** List of 'package' lines describing hashes of downloadable packages */
+  smartlist_t *package_lines;
+
+  /** How long does this vote/consensus claim that authorities take to
+   * distribute their votes to one another? */
+  int vote_seconds;
+  /** How long does this vote/consensus claim that authorities take to
+   * distribute their consensus signatures to one another? */
+  int dist_seconds;
+
+  /** Comma-separated list of recommended client software, or NULL if this
+   * voter has no opinion. */
+  char *client_versions;
+  char *server_versions;
+
+  /** Lists of subprotocol versions which are _recommended_ for relays and
+   * clients, or which are _require_ for relays and clients. Tor shouldn't
+   * make any more network connections if a required protocol is missing.
+   */
+  char *recommended_relay_protocols;
+  char *recommended_client_protocols;
+  char *required_relay_protocols;
+  char *required_client_protocols;
+
+  /** List of flags that this vote/consensus applies to routers.  If a flag is
+   * not listed here, the voter has no opinion on what its value should be. */
+  smartlist_t *known_flags;
+
+  /** List of key=value strings for the parameters in this vote or
+   * consensus, sorted by key. */
+  smartlist_t *net_params;
+
+  /** List of key=value strings for the bw weight parameters in the
+   * consensus. */
+  smartlist_t *weight_params;
+
+  /** List of networkstatus_voter_info_t.  For a vote, only one element
+   * is included.  For a consensus, one element is included for every voter
+   * whose vote contributed to the consensus. */
+  smartlist_t *voters;
+
+  struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */
+
+  /** Digests of this document, as signed. */
+  common_digests_t digests;
+  /** A SHA3-256 digest of the document, not including signatures: used for
+   * consensus diffs */
+  uint8_t digest_sha3_as_signed[DIGEST256_LEN];
+
+  /** List of router statuses, sorted by identity digest.  For a vote,
+   * the elements are vote_routerstatus_t; for a consensus, the elements
+   * are routerstatus_t. */
+  smartlist_t *routerstatus_list;
+
+  /** If present, a map from descriptor digest to elements of
+   * routerstatus_list. */
+  digestmap_t *desc_digest_map;
+
+  /** Contains the shared random protocol data from a vote or consensus. */
+  networkstatus_sr_info_t sr_info;
+};
+
+#endif
diff --git a/src/feature/nodelist/networkstatus_voter_info_st.h b/src/feature/nodelist/networkstatus_voter_info_st.h
new file mode 100644
index 0000000000..93ff3cd418
--- /dev/null
+++ b/src/feature/nodelist/networkstatus_voter_info_st.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NETWORKSTATUS_VOTER_INFO_ST_H
+#define NETWORKSTATUS_VOTER_INFO_ST_H
+
+/** Information about a single voter in a vote or a consensus. */
+struct networkstatus_voter_info_t {
+  /** Declared SHA-1 digest of this voter's identity key */
+  char identity_digest[DIGEST_LEN];
+  char *nickname; /**< Nickname of this voter */
+  /** Digest of this voter's "legacy" identity key, if any.  In vote only; for
+   * consensuses, we treat legacy keys as additional signers. */
+  char legacy_id_digest[DIGEST_LEN];
+  char *address; /**< Address of this voter, in string format. */
+  uint32_t addr; /**< Address of this voter, in IPv4, in host order. */
+  uint16_t dir_port; /**< Directory port of this voter */
+  uint16_t or_port; /**< OR port of this voter */
+  char *contact; /**< Contact information for this voter. */
+  char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */
+
+  /* Nothing from here on is signed. */
+  /** The signature of the document and the signature's status. */
+  smartlist_t *sigs;
+};
+
+#endif
diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h
new file mode 100644
index 0000000000..d56ce27884
--- /dev/null
+++ b/src/feature/nodelist/node_st.h
@@ -0,0 +1,102 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NODE_ST_H
+#define NODE_ST_H
+
+#include "or/hsdir_index_st.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/** A node_t represents a Tor router.
+ *
+ * Specifically, a node_t is a Tor router as we are using it: a router that
+ * we are considering for circuits, connections, and so on.  A node_t is a
+ * thin wrapper around the routerstatus, routerinfo, and microdesc for a
+ * single router, and provides a consistent interface for all of them.
+ *
+ * Also, a node_t has mutable state.  While a routerinfo, a routerstatus,
+ * and a microdesc have[*] only the information read from a router
+ * descriptor, a consensus entry, and a microdescriptor (respectively)...
+ * a node_t has flags based on *our own current opinion* of the node.
+ *
+ * [*] Actually, there is some leftover information in each that is mutable.
+ *  We should try to excise that.
+ */
+struct node_t {
+  /* Indexing information */
+
+  /** Used to look up the node_t by its identity digest. */
+  HT_ENTRY(node_t) ht_ent;
+  /** Used to look up the node_t by its ed25519 identity digest. */
+  HT_ENTRY(node_t) ed_ht_ent;
+  /** Position of the node within the list of nodes */
+  int nodelist_idx;
+
+  /** The identity digest of this node_t.  No more than one node_t per
+   * identity may exist at a time. */
+  char identity[DIGEST_LEN];
+
+  /** The ed25519 identity of this node_t. This field is nonzero iff we
+   * currently have an ed25519 identity for this node in either md or ri,
+   * _and_ this node has been inserted to the ed25519-to-node map in the
+   * nodelist.
+   */
+  ed25519_public_key_t ed25519_id;
+
+  microdesc_t *md;
+  routerinfo_t *ri;
+  routerstatus_t *rs;
+
+  /* local info: copied from routerstatus, then possibly frobbed based
+   * on experience.  Authorities set this stuff directly.  Note that
+   * these reflect knowledge of the primary (IPv4) OR port only.  */
+
+  unsigned int is_running:1; /**< As far as we know, is this OR currently
+                              * running? */
+  unsigned int is_valid:1; /**< Has a trusted dirserver validated this OR?
+                            *  (For Authdir: Have we validated this OR?) */
+  unsigned int is_fast:1; /** Do we think this is a fast OR? */
+  unsigned int is_stable:1; /** Do we think this is a stable OR? */
+  unsigned int is_possible_guard:1; /**< Do we think this is an OK guard? */
+  unsigned int is_exit:1; /**< Do we think this is an OK exit? */
+  unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked,
+                               * or otherwise nasty? */
+  unsigned int is_hs_dir:1; /**< True iff this router is a hidden service
+                             * directory according to the authorities. */
+
+  /* Local info: warning state. */
+
+  unsigned int name_lookup_warned:1; /**< Have we warned the user for referring
+                                      * to this (unnamed) router by nickname?
+                                      */
+
+  /** Local info: we treat this node as if it rejects everything */
+  unsigned int rejects_all:1;
+
+  /* Local info: derived. */
+
+  /** True if the IPv6 OR port is preferred over the IPv4 OR port.
+   * XX/teor - can this become out of date if the torrc changes? */
+  unsigned int ipv6_preferred:1;
+
+  /** According to the geoip db what country is this router in? */
+  /* XXXprop186 what is this suppose to mean with multiple OR ports? */
+  country_t country;
+
+  /* The below items are used only by authdirservers for
+   * reachability testing. */
+
+  /** When was the last time we could reach this OR? */
+  time_t last_reachable;        /* IPv4. */
+  time_t last_reachable6;       /* IPv6. */
+
+  /* Hidden service directory index data. This is used by a service or client
+   * in order to know what's the hs directory index for this node at the time
+   * the consensus is set. */
+  struct hsdir_index_t hsdir_index;
+};
+
+#endif
diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c
new file mode 100644
index 0000000000..51fd0015df
--- /dev/null
+++ b/src/feature/nodelist/nodelist.c
@@ -0,0 +1,2513 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodelist.c
+ *
+ * \brief Structures and functions for tracking what we know about the routers
+ *   on the Tor network, and correlating information from networkstatus,
+ *   routerinfo, and microdescs.
+ *
+ * The key structure here is node_t: that's the canonical way to refer
+ * to a Tor relay that we might want to build a circuit through.  Every
+ * node_t has either a routerinfo_t, or a routerstatus_t from the current
+ * networkstatus consensus.  If it has a routerstatus_t, it will also
+ * need to have a microdesc_t before you can use it for circuits.
+ *
+ * The nodelist_t is a global singleton that maps identities to node_t
+ * objects.  Access them with the node_get_*() functions.  The nodelist_t
+ * is maintained by calls throughout the codebase
+ *
+ * Generally, other code should not have to reach inside a node_t to
+ * see what information it has.  Instead, you should call one of the
+ * many accessor functions that works on a generic node_t.  If there
+ * isn't one that does what you need, it's better to make such a function,
+ * and then use it.
+ *
+ * For historical reasons, some of the functions that select a node_t
+ * from the list of all usable node_t objects are in the routerlist.c
+ * module, since they originally selected a routerinfo_t. (TODO: They
+ * should move!)
+ *
+ * (TODO: Perhaps someday we should abstract the remaining ways of
+ * talking about a relay to also be node_t instances. Those would be
+ * routerstatus_t as used for directory requests, and dir_server_t as
+ * used for authorities and fallback directories.)
+ */
+
+#define NODELIST_PRIVATE
+
+#include "or/or.h"
+#include "lib/net/address.h"
+#include "or/address_set.h"
+#include "or/bridges.h"
+#include "or/config.h"
+#include "or/control.h"
+#include "or/dirserv.h"
+#include "or/entrynodes.h"
+#include "or/geoip.h"
+#include "or/hs_common.h"
+#include "or/hs_client.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/protover.h"
+#include "or/rendservice.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+#include "or/torcert.h"
+
+#include <string.h>
+
+#include "or/dirauth/mode.h"
+
+#include "or/dir_server_st.h"
+#include "or/microdesc_st.h"
+#include "or/networkstatus_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/routerstatus_st.h"
+
+static void nodelist_drop_node(node_t *node, int remove_from_ht);
+#define node_free(val) \
+  FREE_AND_NULL(node_t, node_free_, (val))
+static void node_free_(node_t *node);
+
+/** count_usable_descriptors counts descriptors with these flag(s)
+ */
+typedef enum {
+  /* All descriptors regardless of flags */
+  USABLE_DESCRIPTOR_ALL = 0,
+  /* Only descriptors with the Exit flag */
+  USABLE_DESCRIPTOR_EXIT_ONLY = 1
+} usable_descriptor_t;
+static void count_usable_descriptors(int *num_present,
+                                     int *num_usable,
+                                     smartlist_t *descs_out,
+                                     const networkstatus_t *consensus,
+                                     time_t now,
+                                     routerset_t *in_set,
+                                     usable_descriptor_t exit_only);
+static void update_router_have_minimum_dir_info(void);
+static double get_frac_paths_needed_for_circs(const or_options_t *options,
+                                              const networkstatus_t *ns);
+static void node_add_to_address_set(const node_t *node);
+
+/** A nodelist_t holds a node_t object for every router we're "willing to use
+ * for something".  Specifically, it should hold a node_t for every node that
+ * is currently in the routerlist, or currently in the consensus we're using.
+ */
+typedef struct nodelist_t {
+  /* A list of all the nodes. */
+  smartlist_t *nodes;
+  /* Hash table to map from node ID digest to node. */
+  HT_HEAD(nodelist_map, node_t) nodes_by_id;
+  /* Hash table to map from node Ed25519 ID to node.
+   *
+   * Whenever a node's routerinfo or microdescriptor is about to change,
+   * you should remove it from this map with node_remove_from_ed25519_map().
+   * Whenever a node's routerinfo or microdescriptor has just chaned,
+   * you should add it to this map with node_add_to_ed25519_map().
+   */
+  HT_HEAD(nodelist_ed_map, node_t) nodes_by_ed_id;
+
+  /* Set of addresses that belong to nodes we believe in. */
+  address_set_t *node_addrs;
+
+  /* The valid-after time of the last live consensus that initialized the
+   * nodelist.  We use this to detect outdated nodelists that need to be
+   * rebuilt using a newer consensus. */
+  time_t live_consensus_valid_after;
+} nodelist_t;
+
+static inline unsigned int
+node_id_hash(const node_t *node)
+{
+  return (unsigned) siphash24g(node->identity, DIGEST_LEN);
+}
+
+static inline unsigned int
+node_id_eq(const node_t *node1, const node_t *node2)
+{
+  return tor_memeq(node1->identity, node2->identity, DIGEST_LEN);
+}
+
+HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq)
+HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq,
+             0.6, tor_reallocarray_, tor_free_)
+
+static inline unsigned int
+node_ed_id_hash(const node_t *node)
+{
+  return (unsigned) siphash24g(node->ed25519_id.pubkey, ED25519_PUBKEY_LEN);
+}
+
+static inline unsigned int
+node_ed_id_eq(const node_t *node1, const node_t *node2)
+{
+  return ed25519_pubkey_eq(&node1->ed25519_id, &node2->ed25519_id);
+}
+
+HT_PROTOTYPE(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
+             node_ed_id_eq)
+HT_GENERATE2(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
+             node_ed_id_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/** The global nodelist. */
+static nodelist_t *the_nodelist=NULL;
+
+/** Create an empty nodelist if we haven't done so already. */
+static void
+init_nodelist(void)
+{
+  if (PREDICT_UNLIKELY(the_nodelist == NULL)) {
+    the_nodelist = tor_malloc_zero(sizeof(nodelist_t));
+    HT_INIT(nodelist_map, &the_nodelist->nodes_by_id);
+    HT_INIT(nodelist_ed_map, &the_nodelist->nodes_by_ed_id);
+    the_nodelist->nodes = smartlist_new();
+  }
+}
+
+/** As node_get_by_id, but returns a non-const pointer */
+MOCK_IMPL(node_t *,
+node_get_mutable_by_id,(const char *identity_digest))
+{
+  node_t search, *node;
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return NULL;
+
+  memcpy(&search.identity, identity_digest, DIGEST_LEN);
+  node = HT_FIND(nodelist_map, &the_nodelist->nodes_by_id, &search);
+  return node;
+}
+
+/** As node_get_by_ed25519_id, but returns a non-const pointer */
+node_t *
+node_get_mutable_by_ed25519_id(const ed25519_public_key_t *ed_id)
+{
+  node_t search, *node;
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return NULL;
+  if (BUG(ed_id == NULL) || BUG(ed25519_public_key_is_zero(ed_id)))
+    return NULL;
+
+  memcpy(&search.ed25519_id, ed_id, sizeof(search.ed25519_id));
+  node = HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, &search);
+  return node;
+}
+
+/** Return the node_t whose identity is <b>identity_digest</b>, or NULL
+ * if no such node exists. */
+MOCK_IMPL(const node_t *,
+node_get_by_id,(const char *identity_digest))
+{
+  return node_get_mutable_by_id(identity_digest);
+}
+
+/** Return the node_t whose ed25519 identity is <b>ed_id</b>, or NULL
+ * if no such node exists. */
+MOCK_IMPL(const node_t *,
+node_get_by_ed25519_id,(const ed25519_public_key_t *ed_id))
+{
+  return node_get_mutable_by_ed25519_id(ed_id);
+}
+
+/** Internal: return the node_t whose identity_digest is
+ * <b>identity_digest</b>.  If none exists, create a new one, add it to the
+ * nodelist, and return it.
+ *
+ * Requires that the nodelist be initialized.
+ */
+static node_t *
+node_get_or_create(const char *identity_digest)
+{
+  node_t *node;
+
+  if ((node = node_get_mutable_by_id(identity_digest)))
+    return node;
+
+  node = tor_malloc_zero(sizeof(node_t));
+  memcpy(node->identity, identity_digest, DIGEST_LEN);
+  HT_INSERT(nodelist_map, &the_nodelist->nodes_by_id, node);
+
+  smartlist_add(the_nodelist->nodes, node);
+  node->nodelist_idx = smartlist_len(the_nodelist->nodes) - 1;
+
+  node->country = -1;
+
+  return node;
+}
+
+/** Remove <b>node</b> from the ed25519 map (if it present), and
+ * set its ed25519_id field to zero. */
+static int
+node_remove_from_ed25519_map(node_t *node)
+{
+  tor_assert(the_nodelist);
+  tor_assert(node);
+
+  if (ed25519_public_key_is_zero(&node->ed25519_id)) {
+    return 0;
+  }
+
+  int rv = 0;
+  node_t *search =
+    HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  if (BUG(search != node)) {
+    goto clear_and_return;
+  }
+
+  search = HT_REMOVE(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  tor_assert(search == node);
+  rv = 1;
+
+ clear_and_return:
+  memset(&node->ed25519_id, 0, sizeof(node->ed25519_id));
+  return rv;
+}
+
+/** If <b>node</b> has an ed25519 id, and it is not already in the ed25519 id
+ * map, set its ed25519_id field, and add it to the ed25519 map.
+ */
+static int
+node_add_to_ed25519_map(node_t *node)
+{
+  tor_assert(the_nodelist);
+  tor_assert(node);
+
+  if (! ed25519_public_key_is_zero(&node->ed25519_id)) {
+    return 0;
+  }
+
+  const ed25519_public_key_t *key = node_get_ed25519_id(node);
+  if (!key) {
+    return 0;
+  }
+
+  node_t *old;
+  memcpy(&node->ed25519_id, key, sizeof(node->ed25519_id));
+  old = HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  if (BUG(old)) {
+    /* XXXX order matters here, and this may mean that authorities aren't
+     * pinning. */
+    if (old != node)
+      memset(&node->ed25519_id, 0, sizeof(node->ed25519_id));
+    return 0;
+  }
+
+  HT_INSERT(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  return 1;
+}
+
+/* For a given <b>node</b> for the consensus <b>ns</b>, set the hsdir index
+ * for the node, both current and next if possible. This can only fails if the
+ * node_t ed25519 identity key can't be found which would be a bug. */
+STATIC void
+node_set_hsdir_index(node_t *node, const networkstatus_t *ns)
+{
+  time_t now = approx_time();
+  const ed25519_public_key_t *node_identity_pk;
+  uint8_t *fetch_srv = NULL, *store_first_srv = NULL, *store_second_srv = NULL;
+  uint64_t next_time_period_num, current_time_period_num;
+  uint64_t fetch_tp, store_first_tp, store_second_tp;
+
+  tor_assert(node);
+  tor_assert(ns);
+
+  if (!networkstatus_is_live(ns, now)) {
+    static struct ratelim_t live_consensus_ratelim = RATELIM_INIT(30 * 60);
+    log_fn_ratelim(&live_consensus_ratelim, LOG_INFO, LD_GENERAL,
+                   "Not setting hsdir index with a non-live consensus.");
+    goto done;
+  }
+
+  node_identity_pk = node_get_ed25519_id(node);
+  if (node_identity_pk == NULL) {
+    log_debug(LD_GENERAL, "ed25519 identity public key not found when "
+                          "trying to build the hsdir indexes for node %s",
+              node_describe(node));
+    goto done;
+  }
+
+  /* Get the current and next time period number. */
+  current_time_period_num = hs_get_time_period_num(0);
+  next_time_period_num = hs_get_next_time_period_num(0);
+
+  /* We always use the current time period for fetching descs */
+  fetch_tp = current_time_period_num;
+
+  /* Now extract the needed SRVs and time periods for building hsdir indices */
+  if (hs_in_period_between_tp_and_srv(ns, now)) {
+    fetch_srv = hs_get_current_srv(fetch_tp, ns);
+
+    store_first_tp = hs_get_previous_time_period_num(0);
+    store_second_tp = current_time_period_num;
+  } else {
+    fetch_srv = hs_get_previous_srv(fetch_tp, ns);
+
+    store_first_tp = current_time_period_num;
+    store_second_tp = next_time_period_num;
+  }
+
+  /* We always use the old SRV for storing the first descriptor and the latest
+   * SRV for storing the second descriptor */
+  store_first_srv = hs_get_previous_srv(store_first_tp, ns);
+  store_second_srv = hs_get_current_srv(store_second_tp, ns);
+
+  /* Build the fetch index. */
+  hs_build_hsdir_index(node_identity_pk, fetch_srv, fetch_tp,
+                       node->hsdir_index.fetch);
+
+  /* If we are in the time segment between SRV#N and TP#N, the fetch index is
+     the same as the first store index */
+  if (!hs_in_period_between_tp_and_srv(ns, now)) {
+    memcpy(node->hsdir_index.store_first, node->hsdir_index.fetch,
+           sizeof(node->hsdir_index.store_first));
+  } else {
+    hs_build_hsdir_index(node_identity_pk, store_first_srv, store_first_tp,
+                         node->hsdir_index.store_first);
+  }
+
+  /* If we are in the time segment between TP#N and SRV#N+1, the fetch index is
+     the same as the second store index */
+  if (hs_in_period_between_tp_and_srv(ns, now)) {
+    memcpy(node->hsdir_index.store_second, node->hsdir_index.fetch,
+           sizeof(node->hsdir_index.store_second));
+  } else {
+    hs_build_hsdir_index(node_identity_pk, store_second_srv, store_second_tp,
+                         node->hsdir_index.store_second);
+  }
+
+ done:
+  tor_free(fetch_srv);
+  tor_free(store_first_srv);
+  tor_free(store_second_srv);
+  return;
+}
+
+/** Called when a node's address changes. */
+static void
+node_addrs_changed(node_t *node)
+{
+  node->last_reachable = node->last_reachable6 = 0;
+  node->country = -1;
+}
+
+/** Add all address information about <b>node</b> to the current address
+ * set (if there is one).
+ */
+static void
+node_add_to_address_set(const node_t *node)
+{
+  if (!the_nodelist || !the_nodelist->node_addrs)
+    return;
+
+  /* These various address sources can be redundant, but it's likely faster
+   * to add them all than to compare them all for equality. */
+
+  if (node->rs) {
+    if (node->rs->addr)
+      address_set_add_ipv4h(the_nodelist->node_addrs, node->rs->addr);
+    if (!tor_addr_is_null(&node->rs->ipv6_addr))
+      address_set_add(the_nodelist->node_addrs, &node->rs->ipv6_addr);
+  }
+  if (node->ri) {
+    if (node->ri->addr)
+      address_set_add_ipv4h(the_nodelist->node_addrs, node->ri->addr);
+    if (!tor_addr_is_null(&node->ri->ipv6_addr))
+      address_set_add(the_nodelist->node_addrs, &node->ri->ipv6_addr);
+  }
+  if (node->md) {
+    if (!tor_addr_is_null(&node->md->ipv6_addr))
+      address_set_add(the_nodelist->node_addrs, &node->md->ipv6_addr);
+  }
+}
+
+/** Return true if <b>addr</b> is the address of some node in the nodelist.
+ * If not, probably return false. */
+int
+nodelist_probably_contains_address(const tor_addr_t *addr)
+{
+  if (BUG(!addr))
+    return 0;
+
+  if (!the_nodelist || !the_nodelist->node_addrs)
+    return 0;
+
+  return address_set_probably_contains(the_nodelist->node_addrs, addr);
+}
+
+/** Add <b>ri</b> to an appropriate node in the nodelist.  If we replace an
+ * old routerinfo, and <b>ri_old_out</b> is not NULL, set *<b>ri_old_out</b>
+ * to the previous routerinfo.
+ */
+node_t *
+nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out)
+{
+  node_t *node;
+  const char *id_digest;
+  int had_router = 0;
+  tor_assert(ri);
+
+  init_nodelist();
+  id_digest = ri->cache_info.identity_digest;
+  node = node_get_or_create(id_digest);
+
+  node_remove_from_ed25519_map(node);
+
+  if (node->ri) {
+    if (!routers_have_same_or_addrs(node->ri, ri)) {
+      node_addrs_changed(node);
+    }
+    had_router = 1;
+    if (ri_old_out)
+      *ri_old_out = node->ri;
+  } else {
+    if (ri_old_out)
+      *ri_old_out = NULL;
+  }
+  node->ri = ri;
+
+  node_add_to_ed25519_map(node);
+
+  if (node->country == -1)
+    node_set_country(node);
+
+  if (authdir_mode(get_options()) && !had_router) {
+    const char *discard=NULL;
+    uint32_t status = dirserv_router_get_status(ri, &discard, LOG_INFO);
+    dirserv_set_node_flags_from_authoritative_status(node, status);
+  }
+
+  /* Setting the HSDir index requires the ed25519 identity key which can
+   * only be found either in the ri or md. This is why this is called here.
+   * Only nodes supporting HSDir=2 protocol version needs this index. */
+  if (node->rs && node->rs->pv.supports_v3_hsdir) {
+    node_set_hsdir_index(node,
+                         networkstatus_get_latest_consensus());
+  }
+
+  node_add_to_address_set(node);
+
+  return node;
+}
+
+/** Set the appropriate node_t to use <b>md</b> as its microdescriptor.
+ *
+ * Called when a new microdesc has arrived and the usable consensus flavor
+ * is "microdesc".
+ **/
+node_t *
+nodelist_add_microdesc(microdesc_t *md)
+{
+  networkstatus_t *ns =
+    networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
+  const routerstatus_t *rs;
+  node_t *node;
+  if (ns == NULL)
+    return NULL;
+  init_nodelist();
+
+  /* Microdescriptors don't carry an identity digest, so we need to figure
+   * it out by looking up the routerstatus. */
+  rs = router_get_consensus_status_by_descriptor_digest(ns, md->digest);
+  if (rs == NULL)
+    return NULL;
+  node = node_get_mutable_by_id(rs->identity_digest);
+  if (node == NULL)
+    return NULL;
+
+  node_remove_from_ed25519_map(node);
+  if (node->md)
+    node->md->held_by_nodes--;
+
+  node->md = md;
+  md->held_by_nodes++;
+  /* Setting the HSDir index requires the ed25519 identity key which can
+   * only be found either in the ri or md. This is why this is called here.
+   * Only nodes supporting HSDir=2 protocol version needs this index. */
+  if (rs->pv.supports_v3_hsdir) {
+    node_set_hsdir_index(node, ns);
+  }
+  node_add_to_ed25519_map(node);
+  node_add_to_address_set(node);
+
+  return node;
+}
+
+/* Default value. */
+#define ESTIMATED_ADDRESS_PER_NODE 2
+
+/* Return the estimated number of address per node_t. This is used for the
+ * size of the bloom filter in the nodelist (node_addrs). */
+MOCK_IMPL(int,
+get_estimated_address_per_node, (void))
+{
+  return ESTIMATED_ADDRESS_PER_NODE;
+}
+
+/** Tell the nodelist that the current usable consensus is <b>ns</b>.
+ * This makes the nodelist change all of the routerstatus entries for
+ * the nodes, drop nodes that no longer have enough info to get used,
+ * and grab microdescriptors into nodes as appropriate.
+ */
+void
+nodelist_set_consensus(networkstatus_t *ns)
+{
+  const or_options_t *options = get_options();
+  int authdir = authdir_mode_v3(options);
+
+  init_nodelist();
+  if (ns->flavor == FLAV_MICRODESC)
+    (void) get_microdesc_cache(); /* Make sure it exists first. */
+
+  SMARTLIST_FOREACH(the_nodelist->nodes, node_t *, node,
+                    node->rs = NULL);
+
+  /* Conservatively estimate that every node will have 2 addresses. */
+  const int estimated_addresses = smartlist_len(ns->routerstatus_list) *
+                                  get_estimated_address_per_node();
+  address_set_free(the_nodelist->node_addrs);
+  the_nodelist->node_addrs = address_set_new(estimated_addresses);
+
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    node_t *node = node_get_or_create(rs->identity_digest);
+    node->rs = rs;
+    if (ns->flavor == FLAV_MICRODESC) {
+      if (node->md == NULL ||
+          tor_memneq(node->md->digest,rs->descriptor_digest,DIGEST256_LEN)) {
+        node_remove_from_ed25519_map(node);
+        if (node->md)
+          node->md->held_by_nodes--;
+        node->md = microdesc_cache_lookup_by_digest256(NULL,
+                                                       rs->descriptor_digest);
+        if (node->md)
+          node->md->held_by_nodes++;
+        node_add_to_ed25519_map(node);
+      }
+    }
+
+    if (rs->pv.supports_v3_hsdir) {
+      node_set_hsdir_index(node, ns);
+    }
+    node_set_country(node);
+
+    /* If we're not an authdir, believe others. */
+    if (!authdir) {
+      node->is_valid = rs->is_valid;
+      node->is_running = rs->is_flagged_running;
+      node->is_fast = rs->is_fast;
+      node->is_stable = rs->is_stable;
+      node->is_possible_guard = rs->is_possible_guard;
+      node->is_exit = rs->is_exit;
+      node->is_bad_exit = rs->is_bad_exit;
+      node->is_hs_dir = rs->is_hs_dir;
+      node->ipv6_preferred = 0;
+      if (fascist_firewall_prefer_ipv6_orport(options) &&
+          (tor_addr_is_null(&rs->ipv6_addr) == 0 ||
+           (node->md && tor_addr_is_null(&node->md->ipv6_addr) == 0)))
+        node->ipv6_preferred = 1;
+    }
+
+  } SMARTLIST_FOREACH_END(rs);
+
+  nodelist_purge();
+
+  /* Now add all the nodes we have to the address set. */
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    node_add_to_address_set(node);
+  } SMARTLIST_FOREACH_END(node);
+
+  if (! authdir) {
+    SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+      /* We have no routerstatus for this router. Clear flags so we can skip
+       * it, maybe.*/
+      if (!node->rs) {
+        tor_assert(node->ri); /* if it had only an md, or nothing, purge
+                               * would have removed it. */
+        if (node->ri->purpose == ROUTER_PURPOSE_GENERAL) {
+          /* Clear all flags. */
+          node->is_valid = node->is_running = node->is_hs_dir =
+            node->is_fast = node->is_stable =
+            node->is_possible_guard = node->is_exit =
+            node->is_bad_exit = node->ipv6_preferred = 0;
+        }
+      }
+    } SMARTLIST_FOREACH_END(node);
+  }
+
+  /* If the consensus is live, note down the consensus valid-after that formed
+   * the nodelist. */
+  if (networkstatus_is_live(ns, approx_time())) {
+    the_nodelist->live_consensus_valid_after = ns->valid_after;
+  }
+}
+
+/** Return 1 iff <b>node</b> has Exit flag and no BadExit flag.
+ * Otherwise, return 0.
+ */
+int
+node_is_good_exit(const node_t *node)
+{
+  return node->is_exit && ! node->is_bad_exit;
+}
+
+/** Helper: return true iff a node has a usable amount of information*/
+static inline int
+node_is_usable(const node_t *node)
+{
+  return (node->rs) || (node->ri);
+}
+
+/** Tell the nodelist that <b>md</b> is no longer a microdescriptor for the
+ * node with <b>identity_digest</b>. */
+void
+nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md)
+{
+  node_t *node = node_get_mutable_by_id(identity_digest);
+  if (node && node->md == md) {
+    node->md = NULL;
+    md->held_by_nodes--;
+    if (! node_get_ed25519_id(node)) {
+      node_remove_from_ed25519_map(node);
+    }
+  }
+}
+
+/** Tell the nodelist that <b>ri</b> is no longer in the routerlist. */
+void
+nodelist_remove_routerinfo(routerinfo_t *ri)
+{
+  node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+  if (node && node->ri == ri) {
+    node->ri = NULL;
+    if (! node_is_usable(node)) {
+      nodelist_drop_node(node, 1);
+      node_free(node);
+    }
+  }
+}
+
+/** Remove <b>node</b> from the nodelist.  (Asserts that it was there to begin
+ * with.) */
+static void
+nodelist_drop_node(node_t *node, int remove_from_ht)
+{
+  node_t *tmp;
+  int idx;
+  if (remove_from_ht) {
+    tmp = HT_REMOVE(nodelist_map, &the_nodelist->nodes_by_id, node);
+    tor_assert(tmp == node);
+  }
+  node_remove_from_ed25519_map(node);
+
+  idx = node->nodelist_idx;
+  tor_assert(idx >= 0);
+
+  tor_assert(node == smartlist_get(the_nodelist->nodes, idx));
+  smartlist_del(the_nodelist->nodes, idx);
+  if (idx < smartlist_len(the_nodelist->nodes)) {
+    tmp = smartlist_get(the_nodelist->nodes, idx);
+    tmp->nodelist_idx = idx;
+  }
+  node->nodelist_idx = -1;
+}
+
+/** Return a newly allocated smartlist of the nodes that have <b>md</b> as
+ * their microdescriptor. */
+smartlist_t *
+nodelist_find_nodes_with_microdesc(const microdesc_t *md)
+{
+  smartlist_t *result = smartlist_new();
+
+  if (the_nodelist == NULL)
+    return result;
+
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    if (node->md == md) {
+      smartlist_add(result, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  return result;
+}
+
+/** Release storage held by <b>node</b>  */
+static void
+node_free_(node_t *node)
+{
+  if (!node)
+    return;
+  if (node->md)
+    node->md->held_by_nodes--;
+  tor_assert(node->nodelist_idx == -1);
+  tor_free(node);
+}
+
+/** Remove all entries from the nodelist that don't have enough info to be
+ * usable for anything. */
+void
+nodelist_purge(void)
+{
+  node_t **iter;
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return;
+
+  /* Remove the non-usable nodes. */
+  for (iter = HT_START(nodelist_map, &the_nodelist->nodes_by_id); iter; ) {
+    node_t *node = *iter;
+
+    if (node->md && !node->rs) {
+      /* An md is only useful if there is an rs. */
+      node->md->held_by_nodes--;
+      node->md = NULL;
+    }
+
+    if (node_is_usable(node)) {
+      iter = HT_NEXT(nodelist_map, &the_nodelist->nodes_by_id, iter);
+    } else {
+      iter = HT_NEXT_RMV(nodelist_map, &the_nodelist->nodes_by_id, iter);
+      nodelist_drop_node(node, 0);
+      node_free(node);
+    }
+  }
+  nodelist_assert_ok();
+}
+
+/** Release all storage held by the nodelist. */
+void
+nodelist_free_all(void)
+{
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return;
+
+  HT_CLEAR(nodelist_map, &the_nodelist->nodes_by_id);
+  HT_CLEAR(nodelist_ed_map, &the_nodelist->nodes_by_ed_id);
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    node->nodelist_idx = -1;
+    node_free(node);
+  } SMARTLIST_FOREACH_END(node);
+
+  smartlist_free(the_nodelist->nodes);
+
+  address_set_free(the_nodelist->node_addrs);
+  the_nodelist->node_addrs = NULL;
+
+  tor_free(the_nodelist);
+}
+
+/** Check that the nodelist is internally consistent, and consistent with
+ * the directory info it's derived from.
+ */
+void
+nodelist_assert_ok(void)
+{
+  routerlist_t *rl = router_get_routerlist();
+  networkstatus_t *ns = networkstatus_get_latest_consensus();
+  digestmap_t *dm;
+
+  if (!the_nodelist)
+    return;
+
+  dm = digestmap_new();
+
+  /* every routerinfo in rl->routers should be in the nodelist. */
+  if (rl) {
+    SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) {
+      const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
+      tor_assert(node && node->ri == ri);
+      tor_assert(fast_memeq(ri->cache_info.identity_digest,
+                             node->identity, DIGEST_LEN));
+      tor_assert(! digestmap_get(dm, node->identity));
+      digestmap_set(dm, node->identity, (void*)node);
+    } SMARTLIST_FOREACH_END(ri);
+  }
+
+  /* every routerstatus in ns should be in the nodelist */
+  if (ns) {
+    SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+      const node_t *node = node_get_by_id(rs->identity_digest);
+      tor_assert(node && node->rs == rs);
+      tor_assert(fast_memeq(rs->identity_digest, node->identity, DIGEST_LEN));
+      digestmap_set(dm, node->identity, (void*)node);
+      if (ns->flavor == FLAV_MICRODESC) {
+        /* If it's a microdesc consensus, every entry that has a
+         * microdescriptor should be in the nodelist.
+         */
+        microdesc_t *md =
+          microdesc_cache_lookup_by_digest256(NULL, rs->descriptor_digest);
+        tor_assert(md == node->md);
+        if (md)
+          tor_assert(md->held_by_nodes >= 1);
+      }
+    } SMARTLIST_FOREACH_END(rs);
+  }
+
+  /* The nodelist should have no other entries, and its entries should be
+   * well-formed. */
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    tor_assert(digestmap_get(dm, node->identity) != NULL);
+    tor_assert(node_sl_idx == node->nodelist_idx);
+  } SMARTLIST_FOREACH_END(node);
+
+  /* Every node listed with an ed25519 identity should be listed by that
+   * identity.
+   */
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    if (!ed25519_public_key_is_zero(&node->ed25519_id)) {
+      tor_assert(node == node_get_by_ed25519_id(&node->ed25519_id));
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  node_t **idx;
+  HT_FOREACH(idx, nodelist_ed_map, &the_nodelist->nodes_by_ed_id) {
+    node_t *node = *idx;
+    tor_assert(node == node_get_by_ed25519_id(&node->ed25519_id));
+  }
+
+  tor_assert((long)smartlist_len(the_nodelist->nodes) ==
+             (long)HT_SIZE(&the_nodelist->nodes_by_id));
+
+  tor_assert((long)smartlist_len(the_nodelist->nodes) >=
+             (long)HT_SIZE(&the_nodelist->nodes_by_ed_id));
+
+  digestmap_free(dm, NULL);
+}
+
+/** Ensure that the nodelist has been created with the most recent consensus.
+ *  If that's not the case, make it so.  */
+void
+nodelist_ensure_freshness(networkstatus_t *ns)
+{
+  tor_assert(ns);
+
+  /* We don't even have a nodelist: this is a NOP. */
+  if (!the_nodelist) {
+    return;
+  }
+
+  if (the_nodelist->live_consensus_valid_after != ns->valid_after) {
+    log_info(LD_GENERAL, "Nodelist was not fresh: rebuilding. (%d / %d)",
+             (int) the_nodelist->live_consensus_valid_after,
+             (int) ns->valid_after);
+    nodelist_set_consensus(ns);
+  }
+}
+/** Return a list of a node_t * for every node we know about.  The caller
+ * MUST NOT modify the list. (You can set and clear flags in the nodes if
+ * you must, but you must not add or remove nodes.) */
+MOCK_IMPL(smartlist_t *,
+nodelist_get_list,(void))
+{
+  init_nodelist();
+  return the_nodelist->nodes;
+}
+
+/** Given a hex-encoded nickname of the format DIGEST, $DIGEST, $DIGEST=name,
+ * or $DIGEST~name, return the node with the matching identity digest and
+ * nickname (if any).  Return NULL if no such node exists, or if <b>hex_id</b>
+ * is not well-formed. DOCDOC flags */
+const node_t *
+node_get_by_hex_id(const char *hex_id, unsigned flags)
+{
+  char digest_buf[DIGEST_LEN];
+  char nn_buf[MAX_NICKNAME_LEN+1];
+  char nn_char='\0';
+
+  (void) flags; // XXXX
+
+  if (hex_digest_nickname_decode(hex_id, digest_buf, &nn_char, nn_buf)==0) {
+    const node_t *node = node_get_by_id(digest_buf);
+    if (!node)
+      return NULL;
+    if (nn_char == '=') {
+      /* "=" indicates a Named relay, but there aren't any of those now. */
+      return NULL;
+    }
+    return node;
+  }
+
+  return NULL;
+}
+
+/** Given a nickname (possibly verbose, possibly a hexadecimal digest), return
+ * the corresponding node_t, or NULL if none exists. Warn the user if they
+ * have specified a router by nickname, unless the NNF_NO_WARN_UNNAMED bit is
+ * set in <b>flags</b>. */
+MOCK_IMPL(const node_t *,
+node_get_by_nickname,(const char *nickname, unsigned flags))
+{
+  const int warn_if_unnamed = !(flags & NNF_NO_WARN_UNNAMED);
+
+  if (!the_nodelist)
+    return NULL;
+
+  /* Handle these cases: DIGEST, $DIGEST, $DIGEST=name, $DIGEST~name. */
+  {
+    const node_t *node;
+    if ((node = node_get_by_hex_id(nickname, flags)) != NULL)
+      return node;
+  }
+
+  if (!strcasecmp(nickname, UNNAMED_ROUTER_NICKNAME))
+    return NULL;
+
+  /* Okay, so the name is not canonical for anybody. */
+  {
+    smartlist_t *matches = smartlist_new();
+    const node_t *choice = NULL;
+
+    SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+      if (!strcasecmp(node_get_nickname(node), nickname))
+        smartlist_add(matches, node);
+    } SMARTLIST_FOREACH_END(node);
+
+    if (smartlist_len(matches)>1 && warn_if_unnamed) {
+      int any_unwarned = 0;
+      SMARTLIST_FOREACH_BEGIN(matches, node_t *, node) {
+        if (!node->name_lookup_warned) {
+          node->name_lookup_warned = 1;
+          any_unwarned = 1;
+        }
+      } SMARTLIST_FOREACH_END(node);
+
+      if (any_unwarned) {
+        log_warn(LD_CONFIG, "There are multiple matches for the name %s, "
+                 "but none is listed as Named in the directory consensus. "
+                 "Choosing one arbitrarily.", nickname);
+      }
+    } else if (smartlist_len(matches)==1 && warn_if_unnamed) {
+      char fp[HEX_DIGEST_LEN+1];
+      node_t *node = smartlist_get(matches, 0);
+      if (! node->name_lookup_warned) {
+        base16_encode(fp, sizeof(fp), node->identity, DIGEST_LEN);
+        log_warn(LD_CONFIG,
+                 "You specified a relay \"%s\" by name, but nicknames can be "
+                 "used by any relay, not just the one you meant. "
+                 "To make sure you get the same relay in the future, refer "
+                 "to it by key, as \"$%s\".", nickname, fp);
+        node->name_lookup_warned = 1;
+      }
+    }
+
+    if (smartlist_len(matches))
+      choice = smartlist_get(matches, 0);
+
+    smartlist_free(matches);
+    return choice;
+  }
+}
+
+/** Return the Ed25519 identity key for the provided node, or NULL if it
+ * doesn't have one. */
+const ed25519_public_key_t *
+node_get_ed25519_id(const node_t *node)
+{
+  const ed25519_public_key_t *ri_pk = NULL;
+  const ed25519_public_key_t *md_pk = NULL;
+
+  if (node->ri) {
+    if (node->ri->cache_info.signing_key_cert) {
+      ri_pk = &node->ri->cache_info.signing_key_cert->signing_key;
+      /* Checking whether routerinfo ed25519 is all zero.
+       * Our descriptor parser should make sure this never happens. */
+      if (BUG(ed25519_public_key_is_zero(ri_pk)))
+        ri_pk = NULL;
+    }
+  }
+
+  if (node->md) {
+    if (node->md->ed25519_identity_pkey) {
+      md_pk = node->md->ed25519_identity_pkey;
+      /* Checking whether microdesc ed25519 is all zero.
+       * Our descriptor parser should make sure this never happens. */
+      if (BUG(ed25519_public_key_is_zero(md_pk)))
+        md_pk = NULL;
+    }
+  }
+
+  if (ri_pk && md_pk) {
+    if (ed25519_pubkey_eq(ri_pk, md_pk)) {
+      return ri_pk;
+    } else {
+      /* This can happen if the relay gets flagged NoEdConsensus which will be
+       * triggered on all relays of the network. Thus a protocol warning. */
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Inconsistent ed25519 identities in the nodelist");
+      return NULL;
+    }
+  } else if (ri_pk) {
+    return ri_pk;
+  } else {
+    return md_pk;
+  }
+}
+
+/** Return true iff this node's Ed25519 identity matches <b>id</b>.
+ * (An absent Ed25519 identity matches NULL or zero.) */
+int
+node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id)
+{
+  const ed25519_public_key_t *node_id = node_get_ed25519_id(node);
+  if (node_id == NULL || ed25519_public_key_is_zero(node_id)) {
+    return id == NULL || ed25519_public_key_is_zero(id);
+  } else {
+    return id && ed25519_pubkey_eq(node_id, id);
+  }
+}
+
+/** Dummy object that should be unreturnable.  Used to ensure that
+ * node_get_protover_summary_flags() always returns non-NULL. */
+static const protover_summary_flags_t zero_protover_flags = {
+  0,0,0,0,0,0,0
+};
+
+/** Return the protover_summary_flags for a given node. */
+static const protover_summary_flags_t *
+node_get_protover_summary_flags(const node_t *node)
+{
+  if (node->rs) {
+    return &node->rs->pv;
+  } else if (node->ri) {
+    return &node->ri->pv;
+  } else {
+    /* This should be impossible: every node should have a routerstatus or a
+     * router descriptor or both. But just in case we've messed up somehow,
+     * return a nice empty set of flags to indicate "this node supports
+     * nothing." */
+    tor_assert_nonfatal_unreached_once();
+    return &zero_protover_flags;
+  }
+}
+
+/** Return true iff <b>node</b> supports authenticating itself
+ * by ed25519 ID during the link handshake.  If <b>compatible_with_us</b>,
+ * it needs to be using a link authentication method that we understand.
+ * If not, any plausible link authentication method will do. */
+int
+node_supports_ed25519_link_authentication(const node_t *node,
+                                          int compatible_with_us)
+{
+  if (! node_get_ed25519_id(node))
+    return 0;
+
+  const protover_summary_flags_t *pv = node_get_protover_summary_flags(node);
+
+  if (compatible_with_us)
+    return pv->supports_ed25519_link_handshake_compat;
+  else
+    return pv->supports_ed25519_link_handshake_any;
+}
+
+/** Return true iff <b>node</b> supports the hidden service directory version
+ * 3 protocol (proposal 224). */
+int
+node_supports_v3_hsdir(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_v3_hsdir;
+}
+
+/** Return true iff <b>node</b> supports ed25519 authentication as an hidden
+ * service introduction point.*/
+int
+node_supports_ed25519_hs_intro(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_ed25519_hs_intro;
+}
+
+/** Return true iff <b>node</b> supports to be a rendezvous point for hidden
+ * service version 3 (HSRend=2). */
+int
+node_supports_v3_rendezvous_point(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_v3_rendezvous_point;
+}
+
+/** Return the RSA ID key's SHA1 digest for the provided node. */
+const uint8_t *
+node_get_rsa_id_digest(const node_t *node)
+{
+  tor_assert(node);
+  return (const uint8_t*)node->identity;
+}
+
+/** Return the nickname of <b>node</b>, or NULL if we can't find one. */
+const char *
+node_get_nickname(const node_t *node)
+{
+  tor_assert(node);
+  if (node->rs)
+    return node->rs->nickname;
+  else if (node->ri)
+    return node->ri->nickname;
+  else
+    return NULL;
+}
+
+/** Return true iff <b>node</b> appears to be a directory authority or
+ * directory cache */
+int
+node_is_dir(const node_t *node)
+{
+  if (node->rs) {
+    routerstatus_t * rs = node->rs;
+    /* This is true if supports_tunnelled_dir_requests is true which
+     * indicates that we support directory request tunnelled or through the
+     * DirPort. */
+    return rs->is_v2_dir;
+  } else if (node->ri) {
+    routerinfo_t * ri = node->ri;
+    /* Both tunnelled request is supported or DirPort is set. */
+    return ri->supports_tunnelled_dir_requests;
+  } else {
+    return 0;
+  }
+}
+
+/** Return true iff <b>node</b> has either kind of descriptor -- that
+ * is, a routerdescriptor or a microdescriptor.
+ *
+ * You should probably use node_has_preferred_descriptor() instead.
+ **/
+int
+node_has_any_descriptor(const node_t *node)
+{
+  return (node->ri ||
+          (node->rs && node->md));
+}
+
+/** Return true iff <b>node</b> has the kind of descriptor we would prefer to
+ * use for it, given our configuration and how we intend to use the node.
+ *
+ * If <b>for_direct_connect</b> is true, we intend to connect to the node
+ * directly, as the first hop of a circuit; otherwise, we intend to connect to
+ * it indirectly, or use it as if we were connecting to it indirectly. */
+int
+node_has_preferred_descriptor(const node_t *node,
+                              int for_direct_connect)
+{
+  const int is_bridge = node_is_a_configured_bridge(node);
+  const int we_use_mds = we_use_microdescriptors_for_circuits(get_options());
+
+  if ((is_bridge && for_direct_connect) || !we_use_mds) {
+    /* We need an ri in this case. */
+    if (!node->ri)
+      return 0;
+  } else {
+    /* Otherwise we need an rs and an md. */
+    if (node->rs == NULL || node->md == NULL)
+      return 0;
+  }
+
+  return 1;
+}
+
+/** Return the router_purpose of <b>node</b>. */
+int
+node_get_purpose(const node_t *node)
+{
+  if (node->ri)
+    return node->ri->purpose;
+  else
+    return ROUTER_PURPOSE_GENERAL;
+}
+
+/** Compute the verbose ("extended") nickname of <b>node</b> and store it
+ * into the MAX_VERBOSE_NICKNAME_LEN+1 character buffer at
+ * <b>verbose_name_out</b> */
+void
+node_get_verbose_nickname(const node_t *node,
+                          char *verbose_name_out)
+{
+  const char *nickname = node_get_nickname(node);
+  verbose_name_out[0] = '$';
+  base16_encode(verbose_name_out+1, HEX_DIGEST_LEN+1, node->identity,
+                DIGEST_LEN);
+  if (!nickname)
+    return;
+  verbose_name_out[1+HEX_DIGEST_LEN] = '~';
+  strlcpy(verbose_name_out+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
+}
+
+/** Compute the verbose ("extended") nickname of node with
+ * given <b>id_digest</b> and store it into the MAX_VERBOSE_NICKNAME_LEN+1
+ * character buffer at <b>verbose_name_out</b>
+ *
+ * If node_get_by_id() returns NULL, base 16 encoding of
+ * <b>id_digest</b> is returned instead. */
+void
+node_get_verbose_nickname_by_id(const char *id_digest,
+                                char *verbose_name_out)
+{
+  const node_t *node = node_get_by_id(id_digest);
+  if (!node) {
+    verbose_name_out[0] = '$';
+    base16_encode(verbose_name_out+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
+  } else {
+    node_get_verbose_nickname(node, verbose_name_out);
+  }
+}
+
+/** Return true iff it seems that <b>node</b> allows circuits to exit
+ * through it directlry from the client. */
+int
+node_allows_single_hop_exits(const node_t *node)
+{
+  if (node && node->ri)
+    return node->ri->allow_single_hop_exits;
+  else
+    return 0;
+}
+
+/** Return true iff it seems that <b>node</b> has an exit policy that doesn't
+ * actually permit anything to exit, or we don't know its exit policy */
+int
+node_exit_policy_rejects_all(const node_t *node)
+{
+  if (node->rejects_all)
+    return 1;
+
+  if (node->ri)
+    return node->ri->policy_is_reject_star;
+  else if (node->md)
+    return node->md->exit_policy == NULL ||
+      short_policy_is_reject_star(node->md->exit_policy);
+  else
+    return 1;
+}
+
+/** Return true iff the exit policy for <b>node</b> is such that we can treat
+ * rejecting an address of type <b>family</b> unexpectedly as a sign of that
+ * node's failure. */
+int
+node_exit_policy_is_exact(const node_t *node, sa_family_t family)
+{
+  if (family == AF_UNSPEC) {
+    return 1; /* Rejecting an address but not telling us what address
+               * is a bad sign. */
+  } else if (family == AF_INET) {
+    return node->ri != NULL;
+  } else if (family == AF_INET6) {
+    return 0;
+  }
+  tor_fragile_assert();
+  return 1;
+}
+
+/* Check if the "addr" and port_field fields from r are a valid non-listening
+ * address/port. If so, set valid to true and add a newly allocated
+ * tor_addr_port_t containing "addr" and port_field to sl.
+ * "addr" is an IPv4 host-order address and port_field is a uint16_t.
+ * r is typically a routerinfo_t or routerstatus_t.
+ */
+#define SL_ADD_NEW_IPV4_AP(r, port_field, sl, valid) \
+  STMT_BEGIN \
+    if (tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \
+      valid = 1; \
+      tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); \
+      tor_addr_from_ipv4h(&ap->addr, (r)->addr); \
+      ap->port = (r)->port_field; \
+      smartlist_add((sl), ap); \
+    } \
+  STMT_END
+
+/* Check if the "addr" and port_field fields from r are a valid non-listening
+ * address/port. If so, set valid to true and add a newly allocated
+ * tor_addr_port_t containing "addr" and port_field to sl.
+ * "addr" is a tor_addr_t and port_field is a uint16_t.
+ * r is typically a routerinfo_t or routerstatus_t.
+ */
+#define SL_ADD_NEW_IPV6_AP(r, port_field, sl, valid) \
+  STMT_BEGIN \
+    if (tor_addr_port_is_valid(&(r)->ipv6_addr, (r)->port_field, 0)) { \
+      valid = 1; \
+      tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); \
+      tor_addr_copy(&ap->addr, &(r)->ipv6_addr); \
+      ap->port = (r)->port_field; \
+      smartlist_add((sl), ap); \
+    } \
+  STMT_END
+
+/** Return list of tor_addr_port_t with all OR ports (in the sense IP
+ * addr + TCP port) for <b>node</b>.  Caller must free all elements
+ * using tor_free() and free the list using smartlist_free().
+ *
+ * XXX this is potentially a memory fragmentation hog -- if on
+ * critical path consider the option of having the caller allocate the
+ * memory
+ */
+smartlist_t *
+node_get_all_orports(const node_t *node)
+{
+  smartlist_t *sl = smartlist_new();
+  int valid = 0;
+
+  /* Find a valid IPv4 address and port */
+  if (node->ri != NULL) {
+    SL_ADD_NEW_IPV4_AP(node->ri, or_port, sl, valid);
+  }
+
+  /* If we didn't find a valid address/port in the ri, try the rs */
+  if (!valid && node->rs != NULL) {
+    SL_ADD_NEW_IPV4_AP(node->rs, or_port, sl, valid);
+  }
+
+  /* Find a valid IPv6 address and port */
+  valid = 0;
+  if (node->ri != NULL) {
+    SL_ADD_NEW_IPV6_AP(node->ri, ipv6_orport, sl, valid);
+  }
+
+  if (!valid && node->rs != NULL) {
+    SL_ADD_NEW_IPV6_AP(node->rs, ipv6_orport, sl, valid);
+  }
+
+  if (!valid && node->md != NULL) {
+    SL_ADD_NEW_IPV6_AP(node->md, ipv6_orport, sl, valid);
+  }
+
+  return sl;
+}
+
+#undef SL_ADD_NEW_IPV4_AP
+#undef SL_ADD_NEW_IPV6_AP
+
+/** Wrapper around node_get_prim_orport for backward
+    compatibility.  */
+void
+node_get_addr(const node_t *node, tor_addr_t *addr_out)
+{
+  tor_addr_port_t ap;
+  node_get_prim_orport(node, &ap);
+  tor_addr_copy(addr_out, &ap.addr);
+}
+
+/** Return the host-order IPv4 address for <b>node</b>, or 0 if it doesn't
+ * seem to have one.  */
+uint32_t
+node_get_prim_addr_ipv4h(const node_t *node)
+{
+  /* Don't check the ORPort or DirPort, as this function isn't port-specific,
+   * and the node might have a valid IPv4 address, yet have a zero
+   * ORPort or DirPort.
+   */
+  if (node->ri && tor_addr_is_valid_ipv4h(node->ri->addr, 0)) {
+    return node->ri->addr;
+  } else if (node->rs && tor_addr_is_valid_ipv4h(node->rs->addr, 0)) {
+    return node->rs->addr;
+  }
+  return 0;
+}
+
+/** Copy a string representation of an IP address for <b>node</b> into
+ * the <b>len</b>-byte buffer at <b>buf</b>.  */
+void
+node_get_address_string(const node_t *node, char *buf, size_t len)
+{
+  uint32_t ipv4_addr = node_get_prim_addr_ipv4h(node);
+
+  if (tor_addr_is_valid_ipv4h(ipv4_addr, 0)) {
+    tor_addr_t addr;
+    tor_addr_from_ipv4h(&addr, ipv4_addr);
+    tor_addr_to_str(buf, &addr, len, 0);
+  } else if (len > 0) {
+    buf[0] = '\0';
+  }
+}
+
+/** Return <b>node</b>'s declared uptime, or -1 if it doesn't seem to have
+ * one. */
+long
+node_get_declared_uptime(const node_t *node)
+{
+  if (node->ri)
+    return node->ri->uptime;
+  else
+    return -1;
+}
+
+/** Return <b>node</b>'s platform string, or NULL if we don't know it. */
+const char *
+node_get_platform(const node_t *node)
+{
+  /* If we wanted, we could record the version in the routerstatus_t, since
+   * the consensus lists it.  We don't, though, so this function just won't
+   * work with microdescriptors. */
+  if (node->ri)
+    return node->ri->platform;
+  else
+    return NULL;
+}
+
+/** Return true iff <b>node</b> is one representing this router. */
+int
+node_is_me(const node_t *node)
+{
+  return router_digest_is_me(node->identity);
+}
+
+/** Return <b>node</b> declared family (as a list of names), or NULL if
+ * the node didn't declare a family. */
+const smartlist_t *
+node_get_declared_family(const node_t *node)
+{
+  if (node->ri && node->ri->declared_family)
+    return node->ri->declared_family;
+  else if (node->md && node->md->family)
+    return node->md->family;
+  else
+    return NULL;
+}
+
+/* Does this node have a valid IPv6 address?
+ * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for
+ * checking specific ports. */
+int
+node_has_ipv6_addr(const node_t *node)
+{
+  /* Don't check the ORPort or DirPort, as this function isn't port-specific,
+   * and the node might have a valid IPv6 address, yet have a zero
+   * ORPort or DirPort.
+   */
+  if (node->ri && tor_addr_is_valid(&node->ri->ipv6_addr, 0))
+    return 1;
+  if (node->rs && tor_addr_is_valid(&node->rs->ipv6_addr, 0))
+    return 1;
+  if (node->md && tor_addr_is_valid(&node->md->ipv6_addr, 0))
+    return 1;
+
+  return 0;
+}
+
+/* Does this node have a valid IPv6 ORPort? */
+int
+node_has_ipv6_orport(const node_t *node)
+{
+  tor_addr_port_t ipv6_orport;
+  node_get_pref_ipv6_orport(node, &ipv6_orport);
+  return tor_addr_port_is_valid_ap(&ipv6_orport, 0);
+}
+
+/* Does this node have a valid IPv6 DirPort? */
+int
+node_has_ipv6_dirport(const node_t *node)
+{
+  tor_addr_port_t ipv6_dirport;
+  node_get_pref_ipv6_dirport(node, &ipv6_dirport);
+  return tor_addr_port_is_valid_ap(&ipv6_dirport, 0);
+}
+
+/** Return 1 if we prefer the IPv6 address and OR TCP port of
+ * <b>node</b>, else 0.
+ *
+ *  We prefer the IPv6 address if the router has an IPv6 address,
+ *  and we can use IPv6 addresses, and:
+ *  i) the node_t says that it prefers IPv6
+ *  or
+ *  ii) the router has no IPv4 OR address.
+ *
+ * If you don't have a node, consider looking it up.
+ * If there is no node, use fascist_firewall_prefer_ipv6_orport().
+ */
+int
+node_ipv6_or_preferred(const node_t *node)
+{
+  const or_options_t *options = get_options();
+  tor_addr_port_t ipv4_addr;
+  node_assert_ok(node);
+
+  /* XX/teor - node->ipv6_preferred is set from
+   * fascist_firewall_prefer_ipv6_orport() each time the consensus is loaded.
+   */
+  node_get_prim_orport(node, &ipv4_addr);
+  if (!fascist_firewall_use_ipv6(options)) {
+    return 0;
+  } else if (node->ipv6_preferred ||
+             !tor_addr_port_is_valid_ap(&ipv4_addr, 0)) {
+    return node_has_ipv6_orport(node);
+  }
+  return 0;
+}
+
+#define RETURN_IPV4_AP(r, port_field, ap_out) \
+  STMT_BEGIN \
+    if (r && tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \
+      tor_addr_from_ipv4h(&(ap_out)->addr, (r)->addr); \
+      (ap_out)->port = (r)->port_field; \
+    } \
+  STMT_END
+
+/** Copy the primary (IPv4) OR port (IP address and TCP port) for <b>node</b>
+ * into *<b>ap_out</b>. */
+void
+node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  node_assert_ok(node);
+  tor_assert(ap_out);
+
+  /* Clear the address, as a safety precaution if calling functions ignore the
+   * return value */
+  tor_addr_make_null(&ap_out->addr, AF_INET);
+  ap_out->port = 0;
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address. */
+
+  RETURN_IPV4_AP(node->ri, or_port, ap_out);
+  RETURN_IPV4_AP(node->rs, or_port, ap_out);
+  /* Microdescriptors only have an IPv6 address */
+}
+
+/** Copy the preferred OR port (IP address and TCP port) for
+ * <b>node</b> into *<b>ap_out</b>.  */
+void
+node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  tor_assert(ap_out);
+
+  if (node_ipv6_or_preferred(node)) {
+    node_get_pref_ipv6_orport(node, ap_out);
+  } else {
+    /* the primary ORPort is always on IPv4 */
+    node_get_prim_orport(node, ap_out);
+  }
+}
+
+/** Copy the preferred IPv6 OR port (IP address and TCP port) for
+ * <b>node</b> into *<b>ap_out</b>. */
+void
+node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  node_assert_ok(node);
+  tor_assert(ap_out);
+  memset(ap_out, 0, sizeof(*ap_out));
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
+
+  if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
+                                         node->ri->ipv6_orport, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->ri->ipv6_addr);
+    ap_out->port = node->ri->ipv6_orport;
+  } else if (node->rs && tor_addr_port_is_valid(&node->rs->ipv6_addr,
+                                                 node->rs->ipv6_orport, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->rs->ipv6_addr);
+    ap_out->port = node->rs->ipv6_orport;
+  } else if (node->md && tor_addr_port_is_valid(&node->md->ipv6_addr,
+                                                 node->md->ipv6_orport, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->md->ipv6_addr);
+    ap_out->port = node->md->ipv6_orport;
+  } else {
+    tor_addr_make_null(&ap_out->addr, AF_INET6);
+    ap_out->port = 0;
+  }
+}
+
+/** Return 1 if we prefer the IPv6 address and Dir TCP port of
+ * <b>node</b>, else 0.
+ *
+ *  We prefer the IPv6 address if the router has an IPv6 address,
+ *  and we can use IPv6 addresses, and:
+ *  i) the router has no IPv4 Dir address.
+ *  or
+ *  ii) our preference is for IPv6 Dir addresses.
+ *
+ * If there is no node, use fascist_firewall_prefer_ipv6_dirport().
+ */
+int
+node_ipv6_dir_preferred(const node_t *node)
+{
+  const or_options_t *options = get_options();
+  tor_addr_port_t ipv4_addr;
+  node_assert_ok(node);
+
+  /* node->ipv6_preferred is set from fascist_firewall_prefer_ipv6_orport(),
+   * so we can't use it to determine DirPort IPv6 preference.
+   * This means that bridge clients will use IPv4 DirPorts by default.
+   */
+  node_get_prim_dirport(node, &ipv4_addr);
+  if (!fascist_firewall_use_ipv6(options)) {
+    return 0;
+  } else if (!tor_addr_port_is_valid_ap(&ipv4_addr, 0)
+      || fascist_firewall_prefer_ipv6_dirport(get_options())) {
+    return node_has_ipv6_dirport(node);
+  }
+  return 0;
+}
+
+/** Copy the primary (IPv4) Dir port (IP address and TCP port) for <b>node</b>
+ * into *<b>ap_out</b>. */
+void
+node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  node_assert_ok(node);
+  tor_assert(ap_out);
+
+  /* Clear the address, as a safety precaution if calling functions ignore the
+   * return value */
+  tor_addr_make_null(&ap_out->addr, AF_INET);
+  ap_out->port = 0;
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address. */
+
+  RETURN_IPV4_AP(node->ri, dir_port, ap_out);
+  RETURN_IPV4_AP(node->rs, dir_port, ap_out);
+  /* Microdescriptors only have an IPv6 address */
+}
+
+#undef RETURN_IPV4_AP
+
+/** Copy the preferred Dir port (IP address and TCP port) for
+ * <b>node</b> into *<b>ap_out</b>.  */
+void
+node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  tor_assert(ap_out);
+
+  if (node_ipv6_dir_preferred(node)) {
+    node_get_pref_ipv6_dirport(node, ap_out);
+  } else {
+    /* the primary DirPort is always on IPv4 */
+    node_get_prim_dirport(node, ap_out);
+  }
+}
+
+/** Copy the preferred IPv6 Dir port (IP address and TCP port) for
+ * <b>node</b> into *<b>ap_out</b>. */
+void
+node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out)
+{
+  node_assert_ok(node);
+  tor_assert(ap_out);
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
+
+  /* Assume IPv4 and IPv6 dirports are the same */
+  if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
+                                         node->ri->dir_port, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->ri->ipv6_addr);
+    ap_out->port = node->ri->dir_port;
+  } else if (node->rs && tor_addr_port_is_valid(&node->rs->ipv6_addr,
+                                                node->rs->dir_port, 0)) {
+    tor_addr_copy(&ap_out->addr, &node->rs->ipv6_addr);
+    ap_out->port = node->rs->dir_port;
+  } else {
+    tor_addr_make_null(&ap_out->addr, AF_INET6);
+    ap_out->port = 0;
+  }
+}
+
+/** Return true iff <b>md</b> has a curve25519 onion key.
+ * Use node_has_curve25519_onion_key() instead of calling this directly. */
+static int
+microdesc_has_curve25519_onion_key(const microdesc_t *md)
+{
+  if (!md) {
+    return 0;
+  }
+
+  if (!md->onion_curve25519_pkey) {
+    return 0;
+  }
+
+  if (tor_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
+                      CURVE25519_PUBKEY_LEN)) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/** Return true iff <b>node</b> has a curve25519 onion key. */
+int
+node_has_curve25519_onion_key(const node_t *node)
+{
+  return node_get_curve25519_onion_key(node) != NULL;
+}
+
+/** Return the curve25519 key of <b>node</b>, or NULL if none. */
+const curve25519_public_key_t *
+node_get_curve25519_onion_key(const node_t *node)
+{
+  if (!node)
+    return NULL;
+  if (routerinfo_has_curve25519_onion_key(node->ri))
+    return node->ri->onion_curve25519_pkey;
+  else if (microdesc_has_curve25519_onion_key(node->md))
+    return node->md->onion_curve25519_pkey;
+  else
+    return NULL;
+}
+
+/** Refresh the country code of <b>ri</b>.  This function MUST be called on
+ * each router when the GeoIP database is reloaded, and on all new routers. */
+void
+node_set_country(node_t *node)
+{
+  tor_addr_t addr = TOR_ADDR_NULL;
+
+  /* XXXXipv6 */
+  if (node->rs)
+    tor_addr_from_ipv4h(&addr, node->rs->addr);
+  else if (node->ri)
+    tor_addr_from_ipv4h(&addr, node->ri->addr);
+
+  node->country = geoip_get_country_by_addr(&addr);
+}
+
+/** Set the country code of all routers in the routerlist. */
+void
+nodelist_refresh_countries(void)
+{
+  smartlist_t *nodes = nodelist_get_list();
+  SMARTLIST_FOREACH(nodes, node_t *, node,
+                    node_set_country(node));
+}
+
+/** Return true iff router1 and router2 have similar enough network addresses
+ * that we should treat them as being in the same family */
+int
+addrs_in_same_network_family(const tor_addr_t *a1,
+                             const tor_addr_t *a2)
+{
+  return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC);
+}
+
+/** Return true if <b>node</b>'s nickname matches <b>nickname</b>
+ * (case-insensitive), or if <b>node's</b> identity key digest
+ * matches a hexadecimal value stored in <b>nickname</b>.  Return
+ * false otherwise. */
+static int
+node_nickname_matches(const node_t *node, const char *nickname)
+{
+  const char *n = node_get_nickname(node);
+  if (n && nickname[0]!='$' && !strcasecmp(n, nickname))
+    return 1;
+  return hex_digest_nickname_matches(nickname,
+                                     node->identity,
+                                     n);
+}
+
+/** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */
+static inline int
+node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
+{
+  if (!lst) return 0;
+  SMARTLIST_FOREACH(lst, const char *, name, {
+    if (node_nickname_matches(node, name))
+      return 1;
+  });
+  return 0;
+}
+
+/** Return true iff r1 and r2 are in the same family, but not the same
+ * router. */
+int
+nodes_in_same_family(const node_t *node1, const node_t *node2)
+{
+  const or_options_t *options = get_options();
+
+  /* Are they in the same family because of their addresses? */
+  if (options->EnforceDistinctSubnets) {
+    tor_addr_t a1, a2;
+    node_get_addr(node1, &a1);
+    node_get_addr(node2, &a2);
+    if (addrs_in_same_network_family(&a1, &a2))
+      return 1;
+  }
+
+  /* Are they in the same family because the agree they are? */
+  {
+    const smartlist_t *f1, *f2;
+    f1 = node_get_declared_family(node1);
+    f2 = node_get_declared_family(node2);
+    if (f1 && f2 &&
+        node_in_nickname_smartlist(f1, node2) &&
+        node_in_nickname_smartlist(f2, node1))
+      return 1;
+  }
+
+  /* Are they in the same option because the user says they are? */
+  if (options->NodeFamilySets) {
+    SMARTLIST_FOREACH(options->NodeFamilySets, const routerset_t *, rs, {
+        if (routerset_contains_node(rs, node1) &&
+            routerset_contains_node(rs, node2))
+          return 1;
+      });
+  }
+
+  return 0;
+}
+
+/**
+ * Add all the family of <b>node</b>, including <b>node</b> itself, to
+ * the smartlist <b>sl</b>.
+ *
+ * This is used to make sure we don't pick siblings in a single path, or
+ * pick more than one relay from a family for our entry guard list.
+ * Note that a node may be added to <b>sl</b> more than once if it is
+ * part of <b>node</b>'s family for more than one reason.
+ */
+void
+nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
+{
+  const smartlist_t *all_nodes = nodelist_get_list();
+  const smartlist_t *declared_family;
+  const or_options_t *options = get_options();
+
+  tor_assert(node);
+
+  declared_family = node_get_declared_family(node);
+
+  /* Let's make sure that we have the node itself, if it's a real node. */
+  {
+    const node_t *real_node = node_get_by_id(node->identity);
+    if (real_node)
+      smartlist_add(sl, (node_t*)real_node);
+  }
+
+  /* First, add any nodes with similar network addresses. */
+  if (options->EnforceDistinctSubnets) {
+    tor_addr_t node_addr;
+    node_get_addr(node, &node_addr);
+
+    SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) {
+      tor_addr_t a;
+      node_get_addr(node2, &a);
+      if (addrs_in_same_network_family(&a, &node_addr))
+        smartlist_add(sl, (void*)node2);
+    } SMARTLIST_FOREACH_END(node2);
+  }
+
+  /* Now, add all nodes in the declared_family of this node, if they
+   * also declare this node to be in their family. */
+  if (declared_family) {
+    /* Add every r such that router declares familyness with node, and node
+     * declares familyhood with router. */
+    SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) {
+      const node_t *node2;
+      const smartlist_t *family2;
+      if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED)))
+        continue;
+      if (!(family2 = node_get_declared_family(node2)))
+        continue;
+      SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) {
+          if (node_nickname_matches(node, name2)) {
+            smartlist_add(sl, (void*)node2);
+            break;
+          }
+      } SMARTLIST_FOREACH_END(name2);
+    } SMARTLIST_FOREACH_END(name);
+  }
+
+  /* If the user declared any families locally, honor those too. */
+  if (options->NodeFamilySets) {
+    SMARTLIST_FOREACH(options->NodeFamilySets, const routerset_t *, rs, {
+      if (routerset_contains_node(rs, node)) {
+        routerset_get_all_nodes(sl, rs, NULL, 0);
+      }
+    });
+  }
+}
+
+/** Find a router that's up, that has this IP address, and
+ * that allows exit to this address:port, or return NULL if there
+ * isn't a good one.
+ * Don't exit enclave to excluded relays -- it wouldn't actually
+ * hurt anything, but this way there are fewer confused users.
+ */
+const node_t *
+router_find_exact_exit_enclave(const char *address, uint16_t port)
+{/*XXXX MOVE*/
+  uint32_t addr;
+  struct in_addr in;
+  tor_addr_t a;
+  const or_options_t *options = get_options();
+
+  if (!tor_inet_aton(address, &in))
+    return NULL; /* it's not an IP already */
+  addr = ntohl(in.s_addr);
+
+  tor_addr_from_ipv4h(&a, addr);
+
+  SMARTLIST_FOREACH(nodelist_get_list(), const node_t *, node, {
+    if (node_get_addr_ipv4h(node) == addr &&
+        node->is_running &&
+        compare_tor_addr_to_node_policy(&a, port, node) ==
+          ADDR_POLICY_ACCEPTED &&
+        !routerset_contains_node(options->ExcludeExitNodesUnion_, node))
+      return node;
+  });
+  return NULL;
+}
+
+/** Return 1 if <b>router</b> is not suitable for these parameters, else 0.
+ * If <b>need_uptime</b> is non-zero, we require a minimum uptime.
+ * If <b>need_capacity</b> is non-zero, we require a minimum advertised
+ * bandwidth.
+ * If <b>need_guard</b>, we require that the router is a possible entry guard.
+ */
+int
+node_is_unreliable(const node_t *node, int need_uptime,
+                   int need_capacity, int need_guard)
+{
+  if (need_uptime && !node->is_stable)
+    return 1;
+  if (need_capacity && !node->is_fast)
+    return 1;
+  if (need_guard && !node->is_possible_guard)
+    return 1;
+  return 0;
+}
+
+/** Return 1 if all running sufficiently-stable routers we can use will reject
+ * addr:port. Return 0 if any might accept it. */
+int
+router_exit_policy_all_nodes_reject(const tor_addr_t *addr, uint16_t port,
+                                    int need_uptime)
+{
+  addr_policy_result_t r;
+
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
+    if (node->is_running &&
+        !node_is_unreliable(node, need_uptime, 0, 0)) {
+
+      r = compare_tor_addr_to_node_policy(addr, port, node);
+
+      if (r != ADDR_POLICY_REJECTED && r != ADDR_POLICY_PROBABLY_REJECTED)
+        return 0; /* this one could be ok. good enough. */
+    }
+  } SMARTLIST_FOREACH_END(node);
+  return 1; /* all will reject. */
+}
+
+/** Mark the router with ID <b>digest</b> as running or non-running
+ * in our routerlist. */
+void
+router_set_status(const char *digest, int up)
+{
+  node_t *node;
+  tor_assert(digest);
+
+  SMARTLIST_FOREACH(router_get_fallback_dir_servers(),
+                    dir_server_t *, d,
+                    if (tor_memeq(d->digest, digest, DIGEST_LEN))
+                      d->is_running = up);
+
+  SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
+                    dir_server_t *, d,
+                    if (tor_memeq(d->digest, digest, DIGEST_LEN))
+                      d->is_running = up);
+
+  node = node_get_mutable_by_id(digest);
+  if (node) {
+#if 0
+    log_debug(LD_DIR,"Marking router %s as %s.",
+              node_describe(node), up ? "up" : "down");
+#endif
+    if (!up && node_is_me(node) && !net_is_disabled())
+      log_warn(LD_NET, "We just marked ourself as down. Are your external "
+               "addresses reachable?");
+
+    if (bool_neq(node->is_running, up))
+      router_dir_info_changed();
+
+    node->is_running = up;
+  }
+}
+
+/** True iff, the last time we checked whether we had enough directory info
+ * to build circuits, the answer was "yes". If there are no exits in the
+ * consensus, we act as if we have 100% of the exit directory info. */
+static int have_min_dir_info = 0;
+
+/** Does the consensus contain nodes that can exit? */
+static consensus_path_type_t have_consensus_path = CONSENSUS_PATH_UNKNOWN;
+
+/** True iff enough has changed since the last time we checked whether we had
+ * enough directory info to build circuits that our old answer can no longer
+ * be trusted. */
+static int need_to_update_have_min_dir_info = 1;
+/** String describing what we're missing before we have enough directory
+ * info. */
+static char dir_info_status[512] = "";
+
+/** Return true iff we have enough consensus information to
+ * start building circuits.  Right now, this means "a consensus that's
+ * less than a day old, and at least 60% of router descriptors (configurable),
+ * weighted by bandwidth. Treat the exit fraction as 100% if there are
+ * no exits in the consensus."
+ * To obtain the final weighted bandwidth, we multiply the
+ * weighted bandwidth fraction for each position (guard, middle, exit). */
+MOCK_IMPL(int,
+router_have_minimum_dir_info,(void))
+{
+  static int logged_delay=0;
+  const char *delay_fetches_msg = NULL;
+  if (should_delay_dir_fetches(get_options(), &delay_fetches_msg)) {
+    if (!logged_delay)
+      log_notice(LD_DIR, "Delaying directory fetches: %s", delay_fetches_msg);
+    logged_delay=1;
+    strlcpy(dir_info_status, delay_fetches_msg,  sizeof(dir_info_status));
+    return 0;
+  }
+  logged_delay = 0; /* reset it if we get this far */
+
+  if (PREDICT_UNLIKELY(need_to_update_have_min_dir_info)) {
+    update_router_have_minimum_dir_info();
+  }
+
+  return have_min_dir_info;
+}
+
+/** Set to CONSENSUS_PATH_EXIT if there is at least one exit node
+ * in the consensus. We update this flag in compute_frac_paths_available if
+ * there is at least one relay that has an Exit flag in the consensus.
+ * Used to avoid building exit circuits when they will almost certainly fail.
+ * Set to CONSENSUS_PATH_INTERNAL if there are no exits in the consensus.
+ * (This situation typically occurs during bootstrap of a test network.)
+ * Set to CONSENSUS_PATH_UNKNOWN if we have never checked, or have
+ * reason to believe our last known value was invalid or has expired.
+ * If we're in a network with TestingDirAuthVoteExit set,
+ * this can cause router_have_consensus_path() to be set to
+ * CONSENSUS_PATH_EXIT, even if there are no nodes with accept exit policies.
+ */
+MOCK_IMPL(consensus_path_type_t,
+router_have_consensus_path, (void))
+{
+  return have_consensus_path;
+}
+
+/** Called when our internal view of the directory has changed.  This can be
+ * when the authorities change, networkstatuses change, the list of routerdescs
+ * changes, or number of running routers changes.
+ */
+void
+router_dir_info_changed(void)
+{
+  need_to_update_have_min_dir_info = 1;
+  rend_hsdir_routers_changed();
+  hs_service_dir_info_changed();
+  hs_client_dir_info_changed();
+}
+
+/** Return a string describing what we're missing before we have enough
+ * directory info. */
+const char *
+get_dir_info_status_string(void)
+{
+  return dir_info_status;
+}
+
+/** Iterate over the servers listed in <b>consensus</b>, and count how many of
+ * them seem like ones we'd use (store this in *<b>num_usable</b>), and how
+ * many of <em>those</em> we have descriptors for (store this in
+ * *<b>num_present</b>).
+ *
+ * If <b>in_set</b> is non-NULL, only consider those routers in <b>in_set</b>.
+ * If <b>exit_only</b> is USABLE_DESCRIPTOR_EXIT_ONLY, only consider nodes
+ * with the Exit flag.
+ * If *<b>descs_out</b> is present, add a node_t for each usable descriptor
+ * to it.
+ */
+static void
+count_usable_descriptors(int *num_present, int *num_usable,
+                         smartlist_t *descs_out,
+                         const networkstatus_t *consensus,
+                         time_t now,
+                         routerset_t *in_set,
+                         usable_descriptor_t exit_only)
+{
+  const int md = (consensus->flavor == FLAV_MICRODESC);
+  *num_present = 0, *num_usable = 0;
+
+  SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, routerstatus_t *, rs)
+    {
+       const node_t *node = node_get_by_id(rs->identity_digest);
+       if (!node)
+         continue; /* This would be a bug: every entry in the consensus is
+                    * supposed to have a node. */
+       if (exit_only == USABLE_DESCRIPTOR_EXIT_ONLY && ! rs->is_exit)
+         continue;
+       if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1))
+         continue;
+       if (client_would_use_router(rs, now)) {
+         const char * const digest = rs->descriptor_digest;
+         int present;
+         ++*num_usable; /* the consensus says we want it. */
+         if (md)
+           present = NULL != microdesc_cache_lookup_by_digest256(NULL, digest);
+         else
+           present = NULL != router_get_by_descriptor_digest(digest);
+         if (present) {
+           /* we have the descriptor listed in the consensus. */
+           ++*num_present;
+         }
+         if (descs_out)
+           smartlist_add(descs_out, (node_t*)node);
+       }
+     }
+  SMARTLIST_FOREACH_END(rs);
+
+  log_debug(LD_DIR, "%d usable, %d present (%s%s).",
+            *num_usable, *num_present,
+            md ? "microdesc" : "desc",
+            exit_only == USABLE_DESCRIPTOR_EXIT_ONLY ? " exits" : "s");
+}
+
+/** Return an estimate of which fraction of usable paths through the Tor
+ * network we have available for use.  Count how many routers seem like ones
+ * we'd use (store this in *<b>num_usable_out</b>), and how many of
+ * <em>those</em> we have descriptors for (store this in
+ * *<b>num_present_out</b>.)
+ *
+ * If **<b>status_out</b> is present, allocate a new string and print the
+ * available percentages of guard, middle, and exit nodes to it, noting
+ * whether there are exits in the consensus.
+ * If there are no exits in the consensus, we treat the exit fraction as 100%,
+ * but set router_have_consensus_path() so that we can only build internal
+ * paths. */
+static double
+compute_frac_paths_available(const networkstatus_t *consensus,
+                             const or_options_t *options, time_t now,
+                             int *num_present_out, int *num_usable_out,
+                             char **status_out)
+{
+  smartlist_t *guards = smartlist_new();
+  smartlist_t *mid    = smartlist_new();
+  smartlist_t *exits  = smartlist_new();
+  double f_guard, f_mid, f_exit;
+  double f_path = 0.0;
+  /* Used to determine whether there are any exits in the consensus */
+  int np = 0;
+  /* Used to determine whether there are any exits with descriptors */
+  int nu = 0;
+  const int authdir = authdir_mode_v3(options);
+
+  count_usable_descriptors(num_present_out, num_usable_out,
+                           mid, consensus, now, NULL,
+                           USABLE_DESCRIPTOR_ALL);
+  if (options->EntryNodes) {
+    count_usable_descriptors(&np, &nu, guards, consensus, now,
+                             options->EntryNodes, USABLE_DESCRIPTOR_ALL);
+  } else {
+    SMARTLIST_FOREACH(mid, const node_t *, node, {
+      if (authdir) {
+        if (node->rs && node->rs->is_possible_guard)
+          smartlist_add(guards, (node_t*)node);
+      } else {
+        if (node->is_possible_guard)
+          smartlist_add(guards, (node_t*)node);
+      }
+    });
+  }
+
+  /* All nodes with exit flag
+   * If we're in a network with TestingDirAuthVoteExit set,
+   * this can cause false positives on have_consensus_path,
+   * incorrectly setting it to CONSENSUS_PATH_EXIT. This is
+   * an unavoidable feature of forcing authorities to declare
+   * certain nodes as exits.
+   */
+  count_usable_descriptors(&np, &nu, exits, consensus, now,
+                           NULL, USABLE_DESCRIPTOR_EXIT_ONLY);
+  log_debug(LD_NET,
+            "%s: %d present, %d usable",
+            "exits",
+            np,
+            nu);
+
+  /* We need at least 1 exit present in the consensus to consider
+   * building exit paths */
+  /* Update our understanding of whether the consensus has exits */
+  consensus_path_type_t old_have_consensus_path = have_consensus_path;
+  have_consensus_path = ((nu > 0) ?
+                         CONSENSUS_PATH_EXIT :
+                         CONSENSUS_PATH_INTERNAL);
+
+  if (have_consensus_path == CONSENSUS_PATH_INTERNAL
+      && old_have_consensus_path != have_consensus_path) {
+    log_notice(LD_NET,
+               "The current consensus has no exit nodes. "
+               "Tor can only build internal paths, "
+               "such as paths to hidden services.");
+
+    /* However, exit nodes can reachability self-test using this consensus,
+     * join the network, and appear in a later consensus. This will allow
+     * the network to build exit paths, such as paths for world wide web
+     * browsing (as distinct from hidden service web browsing). */
+  }
+
+  f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD, 1);
+  f_mid   = frac_nodes_with_descriptors(mid,    WEIGHT_FOR_MID,   0);
+  f_exit  = frac_nodes_with_descriptors(exits,  WEIGHT_FOR_EXIT,  0);
+
+  /* If we are using bridges and have at least one bridge with a full
+   * descriptor, assume f_guard is 1.0. */
+  if (options->UseBridges && num_bridges_usable(0) > 0)
+    f_guard = 1.0;
+
+  log_debug(LD_NET,
+            "f_guard: %.2f, f_mid: %.2f, f_exit: %.2f",
+             f_guard,
+             f_mid,
+             f_exit);
+
+  smartlist_free(guards);
+  smartlist_free(mid);
+  smartlist_free(exits);
+
+  if (options->ExitNodes) {
+    double f_myexit, f_myexit_unflagged;
+    smartlist_t *myexits= smartlist_new();
+    smartlist_t *myexits_unflagged = smartlist_new();
+
+    /* All nodes with exit flag in ExitNodes option */
+    count_usable_descriptors(&np, &nu, myexits, consensus, now,
+                             options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY);
+    log_debug(LD_NET,
+              "%s: %d present, %d usable",
+              "myexits",
+              np,
+              nu);
+
+    /* Now compute the nodes in the ExitNodes option where which we don't know
+     * what their exit policy is, or we know it permits something. */
+    count_usable_descriptors(&np, &nu, myexits_unflagged,
+                             consensus, now,
+                             options->ExitNodes, USABLE_DESCRIPTOR_ALL);
+    log_debug(LD_NET,
+              "%s: %d present, %d usable",
+              "myexits_unflagged (initial)",
+              np,
+              nu);
+
+    SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) {
+      if (node_has_preferred_descriptor(node, 0) &&
+          node_exit_policy_rejects_all(node)) {
+        SMARTLIST_DEL_CURRENT(myexits_unflagged, node);
+        /* this node is not actually an exit */
+        np--;
+        /* this node is unusable as an exit */
+        nu--;
+      }
+    } SMARTLIST_FOREACH_END(node);
+
+    log_debug(LD_NET,
+              "%s: %d present, %d usable",
+              "myexits_unflagged (final)",
+              np,
+              nu);
+
+    f_myexit= frac_nodes_with_descriptors(myexits, WEIGHT_FOR_EXIT, 0);
+    f_myexit_unflagged=
+              frac_nodes_with_descriptors(myexits_unflagged,
+                                          WEIGHT_FOR_EXIT, 0);
+
+    log_debug(LD_NET,
+              "f_exit: %.2f, f_myexit: %.2f, f_myexit_unflagged: %.2f",
+              f_exit,
+              f_myexit,
+              f_myexit_unflagged);
+
+    /* If our ExitNodes list has eliminated every possible Exit node, and there
+     * were some possible Exit nodes, then instead consider nodes that permit
+     * exiting to some ports. */
+    if (smartlist_len(myexits) == 0 &&
+        smartlist_len(myexits_unflagged)) {
+      f_myexit = f_myexit_unflagged;
+    }
+
+    smartlist_free(myexits);
+    smartlist_free(myexits_unflagged);
+
+    /* This is a tricky point here: we don't want to make it easy for a
+     * directory to trickle exits to us until it learns which exits we have
+     * configured, so require that we have a threshold both of total exits
+     * and usable exits. */
+    if (f_myexit < f_exit)
+      f_exit = f_myexit;
+  }
+
+  /* if the consensus has no exits, treat the exit fraction as 100% */
+  if (router_have_consensus_path() != CONSENSUS_PATH_EXIT) {
+    f_exit = 1.0;
+  }
+
+  f_path = f_guard * f_mid * f_exit;
+
+  if (status_out)
+    tor_asprintf(status_out,
+                 "%d%% of guards bw, "
+                 "%d%% of midpoint bw, and "
+                 "%d%% of exit bw%s = "
+                 "%d%% of path bw",
+                 (int)(f_guard*100),
+                 (int)(f_mid*100),
+                 (int)(f_exit*100),
+                 (router_have_consensus_path() == CONSENSUS_PATH_EXIT ?
+                  "" :
+                  " (no exits in consensus)"),
+                 (int)(f_path*100));
+
+  return f_path;
+}
+
+/** We just fetched a new set of descriptors. Compute how far through
+ * the "loading descriptors" bootstrapping phase we are, so we can inform
+ * the controller of our progress. */
+int
+count_loading_descriptors_progress(void)
+{
+  int num_present = 0, num_usable=0;
+  time_t now = time(NULL);
+  const or_options_t *options = get_options();
+  const networkstatus_t *consensus =
+    networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor());
+  double paths, fraction;
+
+  if (!consensus)
+    return 0; /* can't count descriptors if we have no list of them */
+
+  paths = compute_frac_paths_available(consensus, options, now,
+                                       &num_present, &num_usable,
+                                       NULL);
+
+  fraction = paths / get_frac_paths_needed_for_circs(options,consensus);
+  if (fraction > 1.0)
+    return 0; /* it's not the number of descriptors holding us back */
+  return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int)
+    (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 -
+               BOOTSTRAP_STATUS_LOADING_DESCRIPTORS));
+}
+
+/** Return the fraction of paths needed before we're willing to build
+ * circuits, as configured in <b>options</b>, or in the consensus <b>ns</b>. */
+static double
+get_frac_paths_needed_for_circs(const or_options_t *options,
+                                const networkstatus_t *ns)
+{
+#define DFLT_PCT_USABLE_NEEDED 60
+  if (options->PathsNeededToBuildCircuits >= 0.0) {
+    return options->PathsNeededToBuildCircuits;
+  } else {
+    return networkstatus_get_param(ns, "min_paths_for_circs_pct",
+                                   DFLT_PCT_USABLE_NEEDED,
+                                   25, 95)/100.0;
+  }
+}
+
+/** Change the value of have_min_dir_info, setting it true iff we have enough
+ * network and router information to build circuits.  Clear the value of
+ * need_to_update_have_min_dir_info. */
+static void
+update_router_have_minimum_dir_info(void)
+{
+  time_t now = time(NULL);
+  int res;
+  int num_present=0, num_usable=0;
+  const or_options_t *options = get_options();
+  const networkstatus_t *consensus =
+    networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor());
+  int using_md;
+
+  if (!consensus) {
+    if (!networkstatus_get_latest_consensus())
+      strlcpy(dir_info_status, "We have no usable consensus.",
+              sizeof(dir_info_status));
+    else
+      strlcpy(dir_info_status, "We have no recent usable consensus.",
+              sizeof(dir_info_status));
+    res = 0;
+    goto done;
+  }
+
+  using_md = consensus->flavor == FLAV_MICRODESC;
+
+  /* Check fraction of available paths */
+  {
+    char *status = NULL;
+    double paths = compute_frac_paths_available(consensus, options, now,
+                                                &num_present, &num_usable,
+                                                &status);
+
+    if (paths < get_frac_paths_needed_for_circs(options,consensus)) {
+      tor_snprintf(dir_info_status, sizeof(dir_info_status),
+                   "We need more %sdescriptors: we have %d/%d, and "
+                   "can only build %d%% of likely paths. (We have %s.)",
+                   using_md?"micro":"", num_present, num_usable,
+                   (int)(paths*100), status);
+      tor_free(status);
+      res = 0;
+      control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0);
+      goto done;
+    }
+
+    tor_free(status);
+    res = 1;
+  }
+
+  { /* Check entry guard dirinfo status */
+    char *guard_error = entry_guards_get_err_str_if_dir_info_missing(using_md,
+                                                             num_present,
+                                                             num_usable);
+    if (guard_error) {
+      strlcpy(dir_info_status, guard_error, sizeof(dir_info_status));
+      tor_free(guard_error);
+      res = 0;
+      goto done;
+    }
+  }
+
+ done:
+
+  /* If paths have just become available in this update. */
+  if (res && !have_min_dir_info) {
+    control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO");
+    if (control_event_bootstrap(BOOTSTRAP_STATUS_CONN_OR, 0) == 0) {
+      log_notice(LD_DIR,
+              "We now have enough directory information to build circuits.");
+    }
+  }
+
+  /* If paths have just become unavailable in this update. */
+  if (!res && have_min_dir_info) {
+    int quiet = directory_too_idle_to_fetch_descriptors(options, now);
+    tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR,
+        "Our directory information is no longer up-to-date "
+        "enough to build circuits: %s", dir_info_status);
+
+    /* a) make us log when we next complete a circuit, so we know when Tor
+     * is back up and usable, and b) disable some activities that Tor
+     * should only do while circuits are working, like reachability tests
+     * and fetching bridge descriptors only over circuits. */
+    note_that_we_maybe_cant_complete_circuits();
+    have_consensus_path = CONSENSUS_PATH_UNKNOWN;
+    control_event_client_status(LOG_NOTICE, "NOT_ENOUGH_DIR_INFO");
+  }
+  have_min_dir_info = res;
+  need_to_update_have_min_dir_info = 0;
+}
+
diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h
new file mode 100644
index 0000000000..ed3a542971
--- /dev/null
+++ b/src/feature/nodelist/nodelist.h
@@ -0,0 +1,168 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodelist.h
+ * \brief Header file for nodelist.c.
+ **/
+
+#ifndef TOR_NODELIST_H
+#define TOR_NODELIST_H
+
+struct ed25519_public_key_t;
+struct curve25519_public_key_t;
+
+#define node_assert_ok(n) STMT_BEGIN {                          \
+    tor_assert((n)->ri || (n)->rs);                             \
+  } STMT_END
+
+MOCK_DECL(node_t *, node_get_mutable_by_id,(const char *identity_digest));
+MOCK_DECL(const node_t *, node_get_by_id, (const char *identity_digest));
+node_t *node_get_mutable_by_ed25519_id(
+                            const struct ed25519_public_key_t *ed_id);
+MOCK_DECL(const node_t *, node_get_by_ed25519_id,
+          (const struct ed25519_public_key_t *ed_id));
+
+#define NNF_NO_WARN_UNNAMED (1u<<0)
+
+const node_t *node_get_by_hex_id(const char *identity_digest,
+                                 unsigned flags);
+node_t *nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out);
+node_t *nodelist_add_microdesc(microdesc_t *md);
+void nodelist_set_consensus(networkstatus_t *ns);
+void nodelist_ensure_freshness(networkstatus_t *ns);
+int nodelist_probably_contains_address(const tor_addr_t *addr);
+
+void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md);
+void nodelist_remove_routerinfo(routerinfo_t *ri);
+void nodelist_purge(void);
+smartlist_t *nodelist_find_nodes_with_microdesc(const microdesc_t *md);
+
+void nodelist_free_all(void);
+void nodelist_assert_ok(void);
+
+MOCK_DECL(const node_t *, node_get_by_nickname,
+          (const char *nickname, unsigned flags));
+void node_get_verbose_nickname(const node_t *node,
+                               char *verbose_name_out);
+void node_get_verbose_nickname_by_id(const char *id_digest,
+                                char *verbose_name_out);
+int node_is_dir(const node_t *node);
+int node_is_good_exit(const node_t *node);
+int node_has_any_descriptor(const node_t *node);
+int node_has_preferred_descriptor(const node_t *node,
+                                  int for_direct_connect);
+int node_get_purpose(const node_t *node);
+#define node_is_bridge(node) \
+  (node_get_purpose((node)) == ROUTER_PURPOSE_BRIDGE)
+int node_is_me(const node_t *node);
+int node_exit_policy_rejects_all(const node_t *node);
+int node_exit_policy_is_exact(const node_t *node, sa_family_t family);
+smartlist_t *node_get_all_orports(const node_t *node);
+int node_allows_single_hop_exits(const node_t *node);
+const char *node_get_nickname(const node_t *node);
+const char *node_get_platform(const node_t *node);
+uint32_t node_get_prim_addr_ipv4h(const node_t *node);
+void node_get_address_string(const node_t *node, char *cp, size_t len);
+long node_get_declared_uptime(const node_t *node);
+const smartlist_t *node_get_declared_family(const node_t *node);
+const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node);
+int node_ed25519_id_matches(const node_t *node,
+                            const struct ed25519_public_key_t *id);
+int node_supports_ed25519_link_authentication(const node_t *node,
+                                              int compatible_with_us);
+int node_supports_v3_hsdir(const node_t *node);
+int node_supports_ed25519_hs_intro(const node_t *node);
+int node_supports_v3_rendezvous_point(const node_t *node);
+const uint8_t *node_get_rsa_id_digest(const node_t *node);
+
+int node_has_ipv6_addr(const node_t *node);
+int node_has_ipv6_orport(const node_t *node);
+int node_has_ipv6_dirport(const node_t *node);
+/* Deprecated - use node_ipv6_or_preferred or node_ipv6_dir_preferred */
+#define node_ipv6_preferred(node) node_ipv6_or_preferred(node)
+int node_ipv6_or_preferred(const node_t *node);
+void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out);
+int node_ipv6_dir_preferred(const node_t *node);
+void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out);
+int node_has_curve25519_onion_key(const node_t *node);
+const struct curve25519_public_key_t *node_get_curve25519_onion_key(
+                                  const node_t *node);
+
+MOCK_DECL(smartlist_t *, nodelist_get_list, (void));
+
+/* Temporary during transition to multiple addresses.  */
+void node_get_addr(const node_t *node, tor_addr_t *addr_out);
+#define node_get_addr_ipv4h(n) node_get_prim_addr_ipv4h((n))
+
+void nodelist_refresh_countries(void);
+void node_set_country(node_t *node);
+void nodelist_add_node_and_family(smartlist_t *nodes, const node_t *node);
+int nodes_in_same_family(const node_t *node1, const node_t *node2);
+
+const node_t *router_find_exact_exit_enclave(const char *address,
+                                             uint16_t port);
+int node_is_unreliable(const node_t *router, int need_uptime,
+                         int need_capacity, int need_guard);
+int router_exit_policy_all_nodes_reject(const tor_addr_t *addr, uint16_t port,
+                                        int need_uptime);
+void router_set_status(const char *digest, int up);
+int addrs_in_same_network_family(const tor_addr_t *a1,
+                                 const tor_addr_t *a2);
+
+/** router_have_minimum_dir_info tests to see if we have enough
+ * descriptor information to create circuits.
+ * If there are exits in the consensus, we wait until we have enough
+ * info to create exit paths before creating any circuits. If there are
+ * no exits in the consensus, we wait for enough info to create internal
+ * paths, and should avoid creating exit paths, as they will simply fail.
+ * We make sure we create all available circuit types at the same time. */
+MOCK_DECL(int, router_have_minimum_dir_info,(void));
+
+/** Set to CONSENSUS_PATH_EXIT if there is at least one exit node
+ * in the consensus. We update this flag in compute_frac_paths_available if
+ * there is at least one relay that has an Exit flag in the consensus.
+ * Used to avoid building exit circuits when they will almost certainly fail.
+ * Set to CONSENSUS_PATH_INTERNAL if there are no exits in the consensus.
+ * (This situation typically occurs during bootstrap of a test network.)
+ * Set to CONSENSUS_PATH_UNKNOWN if we have never checked, or have
+ * reason to believe our last known value was invalid or has expired.
+ */
+typedef enum {
+  /* we haven't checked yet, or we have invalidated our previous check */
+  CONSENSUS_PATH_UNKNOWN = -1,
+  /* The consensus only has internal relays, and we should only
+   * create internal paths, circuits, streams, ... */
+  CONSENSUS_PATH_INTERNAL = 0,
+  /* The consensus has at least one exit, and can therefore (potentially)
+   * create exit and internal paths, circuits, streams, ... */
+  CONSENSUS_PATH_EXIT = 1
+} consensus_path_type_t;
+
+MOCK_DECL(consensus_path_type_t, router_have_consensus_path, (void));
+
+void router_dir_info_changed(void);
+const char *get_dir_info_status_string(void);
+int count_loading_descriptors_progress(void);
+
+#ifdef NODELIST_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC void
+node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(NODELIST_PRIVATE) */
+
+MOCK_DECL(int, get_estimated_address_per_node, (void));
+
+#endif /* !defined(TOR_NODELIST_H) */
diff --git a/src/feature/nodelist/parsecommon.c b/src/feature/nodelist/parsecommon.c
new file mode 100644
index 0000000000..4340f28225
--- /dev/null
+++ b/src/feature/nodelist/parsecommon.c
@@ -0,0 +1,458 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.c
+ * \brief Common code to parse and validate various type of descriptors.
+ **/
+
+#include "or/parsecommon.h"
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+#include "lib/encoding/binascii.h"
+#include "lib/container/smartlist.h"
+#include "lib/string/util_string.h"
+#include "lib/string/printf.h"
+#include "lib/memarea/memarea.h"
+#include "lib/crypt_ops/crypto.h"
+
+#include <string.h>
+
+#define MIN_ANNOTATION A_PURPOSE
+#define MAX_ANNOTATION A_UNKNOWN_
+
+#define ALLOC_ZERO(sz) memarea_alloc_zero(area,sz)
+#define ALLOC(sz) memarea_alloc(area,sz)
+#define STRDUP(str) memarea_strdup(area,str)
+#define STRNDUP(str,n) memarea_strndup(area,(str),(n))
+
+#define RET_ERR(msg)                                               \
+  STMT_BEGIN                                                       \
+    if (tok) token_clear(tok);                                      \
+    tok = ALLOC_ZERO(sizeof(directory_token_t));                   \
+    tok->tp = ERR_;                                                \
+    tok->error = STRDUP(msg);                                      \
+    goto done_tokenizing;                                          \
+  STMT_END
+
+/** Free all resources allocated for <b>tok</b> */
+void
+token_clear(directory_token_t *tok)
+{
+  if (tok->key)
+    crypto_pk_free(tok->key);
+}
+
+/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
+ * them to <b>out</b>.  Parse according to the token rules in <b>table</b>.
+ * Caller must free tokens in <b>out</b>.  If <b>end</b> is NULL, use the
+ * entire string.
+ */
+int
+tokenize_string(memarea_t *area,
+                const char *start, const char *end, smartlist_t *out,
+                token_rule_t *table, int flags)
+{
+  const char **s;
+  directory_token_t *tok = NULL;
+  int counts[NIL_];
+  int i;
+  int first_nonannotation;
+  int prev_len = smartlist_len(out);
+  tor_assert(area);
+
+  s = &start;
+  if (!end) {
+    end = start+strlen(start);
+  } else {
+    /* it's only meaningful to check for nuls if we got an end-of-string ptr */
+    if (memchr(start, '\0', end-start)) {
+      log_warn(LD_DIR, "parse error: internal NUL character.");
+      return -1;
+    }
+  }
+  for (i = 0; i < NIL_; ++i)
+    counts[i] = 0;
+
+  SMARTLIST_FOREACH(out, const directory_token_t *, t, ++counts[t->tp]);
+
+  while (*s < end && (!tok || tok->tp != EOF_)) {
+    tok = get_next_token(area, s, end, table);
+    if (tok->tp == ERR_) {
+      log_warn(LD_DIR, "parse error: %s", tok->error);
+      token_clear(tok);
+      return -1;
+    }
+    ++counts[tok->tp];
+    smartlist_add(out, tok);
+    *s = eat_whitespace_eos(*s, end);
+  }
+
+  if (flags & TS_NOCHECK)
+    return 0;
+
+  if ((flags & TS_ANNOTATIONS_OK)) {
+    first_nonannotation = -1;
+    for (i = 0; i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp < MIN_ANNOTATION || tok->tp > MAX_ANNOTATION) {
+        first_nonannotation = i;
+        break;
+      }
+    }
+    if (first_nonannotation < 0) {
+      log_warn(LD_DIR, "parse error: item contains only annotations");
+      return -1;
+    }
+    for (i=first_nonannotation;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: Annotations mixed with keywords");
+        return -1;
+      }
+    }
+    if ((flags & TS_NO_NEW_ANNOTATIONS)) {
+      if (first_nonannotation != prev_len) {
+        log_warn(LD_DIR, "parse error: Unexpected annotations.");
+        return -1;
+      }
+    }
+  } else {
+    for (i=0;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: no annotations allowed.");
+        return -1;
+      }
+    }
+    first_nonannotation = 0;
+  }
+  for (i = 0; table[i].t; ++i) {
+    if (counts[table[i].v] < table[i].min_cnt) {
+      log_warn(LD_DIR, "Parse error: missing %s element.", table[i].t);
+      return -1;
+    }
+    if (counts[table[i].v] > table[i].max_cnt) {
+      log_warn(LD_DIR, "Parse error: too many %s elements.", table[i].t);
+      return -1;
+    }
+    if (table[i].pos & AT_START) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, first_nonannotation))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: first item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+    if (table[i].pos & AT_END) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, smartlist_len(out)-1))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: last item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+/** Helper: parse space-separated arguments from the string <b>s</b> ending at
+ * <b>eol</b>, and store them in the args field of <b>tok</b>.  Store the
+ * number of parsed elements into the n_args field of <b>tok</b>.  Allocate
+ * all storage in <b>area</b>.  Return the number of arguments parsed, or
+ * return -1 if there was an insanely high number of arguments. */
+static inline int
+get_token_arguments(memarea_t *area, directory_token_t *tok,
+                    const char *s, const char *eol)
+{
+/** Largest number of arguments we'll accept to any token, ever. */
+#define MAX_ARGS 512
+  char *mem = memarea_strndup(area, s, eol-s);
+  char *cp = mem;
+  int j = 0;
+  char *args[MAX_ARGS];
+  memset(args, 0, sizeof(args));
+  while (*cp) {
+    if (j == MAX_ARGS)
+      return -1;
+    args[j++] = cp;
+    cp = (char*)find_whitespace(cp);
+    if (!cp || !*cp)
+      break; /* End of the line. */
+    *cp++ = '\0';
+    cp = (char*)eat_whitespace(cp);
+  }
+  tok->n_args = j;
+  tok->args = memarea_memdup(area, args, j*sizeof(char*));
+  return j;
+#undef MAX_ARGS
+}
+
+/** Helper: make sure that the token <b>tok</b> with keyword <b>kwd</b> obeys
+ * the object syntax of <b>o_syn</b>.  Allocate all storage in <b>area</b>.
+ * Return <b>tok</b> on success, or a new ERR_ token if the token didn't
+ * conform to the syntax we wanted.
+ **/
+static inline directory_token_t *
+token_check_object(memarea_t *area, const char *kwd,
+                   directory_token_t *tok, obj_syntax o_syn)
+{
+  char ebuf[128];
+  switch (o_syn) {
+    case NO_OBJ:
+      /* No object is allowed for this token. */
+      if (tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_OBJ:
+      /* There must be a (non-key) object. */
+      if (!tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_KEY_1024: /* There must be a 1024-bit public key. */
+    case NEED_SKEY_1024: /* There must be a 1024-bit private key. */
+      if (tok->key && crypto_pk_num_bits(tok->key) != PK_BYTES*8) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Wrong size on key for %s: %d bits",
+                     kwd, crypto_pk_num_bits(tok->key));
+        RET_ERR(ebuf);
+      }
+      /* fall through */
+    case NEED_KEY: /* There must be some kind of key. */
+      if (!tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (o_syn != NEED_SKEY_1024) {
+        if (crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Private key given for %s, which wants a public key", kwd);
+          RET_ERR(ebuf);
+        }
+      } else { /* o_syn == NEED_SKEY_1024 */
+        if (!crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Public key given for %s, which wants a private key", kwd);
+          RET_ERR(ebuf);
+        }
+      }
+      break;
+    case OBJ_OK:
+      /* Anything goes with this token. */
+      break;
+  }
+
+ done_tokenizing:
+  return tok;
+}
+
+/** Helper function: read the next token from *s, advance *s to the end of the
+ * token, and return the parsed token.  Parse *<b>s</b> according to the list
+ * of tokens in <b>table</b>.
+ */
+directory_token_t *
+get_next_token(memarea_t *area,
+               const char **s, const char *eos, token_rule_t *table)
+{
+  /** Reject any object at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_UNPARSED_OBJECT_SIZE (128*1024)
+  /** Reject any line at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_LINE_LENGTH (128*1024)
+
+  const char *next, *eol, *obstart;
+  size_t obname_len;
+  int i;
+  directory_token_t *tok;
+  obj_syntax o_syn = NO_OBJ;
+  char ebuf[128];
+  const char *kwd = "";
+
+  tor_assert(area);
+  tok = ALLOC_ZERO(sizeof(directory_token_t));
+  tok->tp = ERR_;
+
+  /* Set *s to first token, eol to end-of-line, next to after first token */
+  *s = eat_whitespace_eos(*s, eos); /* eat multi-line whitespace */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol)
+    eol = eos;
+  if (eol - *s > MAX_LINE_LENGTH) {
+    RET_ERR("Line far too long");
+  }
+
+  next = find_whitespace_eos(*s, eol);
+
+  if (!strcmp_len(*s, "opt", next-*s)) {
+    /* Skip past an "opt" at the start of the line. */
+    *s = eat_whitespace_eos_no_nl(next, eol);
+    next = find_whitespace_eos(*s, eol);
+  } else if (*s == eos) {  /* If no "opt", and end-of-line, line is invalid */
+    RET_ERR("Unexpected EOF");
+  }
+
+  /* Search the table for the appropriate entry.  (I tried a binary search
+   * instead, but it wasn't any faster.) */
+  for (i = 0; table[i].t ; ++i) {
+    if (!strcmp_len(*s, table[i].t, next-*s)) {
+      /* We've found the keyword. */
+      kwd = table[i].t;
+      tok->tp = table[i].v;
+      o_syn = table[i].os;
+      *s = eat_whitespace_eos_no_nl(next, eol);
+      /* We go ahead whether there are arguments or not, so that tok->args is
+       * always set if we want arguments. */
+      if (table[i].concat_args) {
+        /* The keyword takes the line as a single argument */
+        tok->args = ALLOC(sizeof(char*));
+        tok->args[0] = STRNDUP(*s,eol-*s); /* Grab everything on line */
+        tok->n_args = 1;
+      } else {
+        /* This keyword takes multiple arguments. */
+        if (get_token_arguments(area, tok, *s, eol)<0) {
+          tor_snprintf(ebuf, sizeof(ebuf),"Far too many arguments to %s", kwd);
+          RET_ERR(ebuf);
+        }
+        *s = eol;
+      }
+      if (tok->n_args < table[i].min_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too few arguments to %s", kwd);
+        RET_ERR(ebuf);
+      } else if (tok->n_args > table[i].max_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too many arguments to %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    }
+  }
+
+  if (tok->tp == ERR_) {
+    /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
+    if (*s < eol && **s == '@')
+      tok->tp = A_UNKNOWN_;
+    else
+      tok->tp = K_OPT;
+    tok->args = ALLOC(sizeof(char*));
+    tok->args[0] = STRNDUP(*s, eol-*s);
+    tok->n_args = 1;
+    o_syn = OBJ_OK;
+  }
+
+  /* Check whether there's an object present */
+  *s = eat_whitespace_eos(eol, eos);  /* Scan from end of first line */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
+    goto check_object;
+
+  obstart = *s; /* Set obstart to start of object spec */
+  if (*s+16 >= eol || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
+      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
+      (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
+    RET_ERR("Malformed object: bad begin line");
+  }
+  tok->object_type = STRNDUP(*s+11, eol-*s-16);
+  obname_len = eol-*s-16; /* store objname length here to avoid a strlen() */
+  *s = eol+1;    /* Set *s to possible start of object data (could be eos) */
+
+  /* Go to the end of the object */
+  next = tor_memstr(*s, eos-*s, "-----END ");
+  if (!next) {
+    RET_ERR("Malformed object: missing object end line");
+  }
+  tor_assert(eos >= next);
+  eol = memchr(next, '\n', eos-next);
+  if (!eol)  /* end-of-line marker, or eos if there's no '\n' */
+    eol = eos;
+  /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
+  if ((size_t)(eol-next) != 9+obname_len+5 ||
+      strcmp_len(next+9, tok->object_type, obname_len) ||
+      strcmp_len(eol-5, "-----", 5)) {
+    tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
+             tok->object_type);
+    ebuf[sizeof(ebuf)-1] = '\0';
+    RET_ERR(ebuf);
+  }
+  if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
+    RET_ERR("Couldn't parse object: missing footer or object much too big.");
+
+  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse public key.");
+  } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse private key.");
+  } else { /* If it's something else, try to base64-decode it */
+    int r;
+    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
+    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
+    if (r<0)
+      RET_ERR("Malformed object: bad base64-encoded data");
+    tok->object_size = r;
+  }
+  *s = eol;
+
+ check_object:
+  tok = token_check_object(area, kwd, tok, o_syn);
+
+ done_tokenizing:
+  return tok;
+
+#undef RET_ERR
+#undef ALLOC
+#undef ALLOC_ZERO
+#undef STRDUP
+#undef STRNDUP
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; fail
+ * with an assert if no such keyword is found.
+ */
+directory_token_t *
+find_by_keyword_(smartlist_t *s, directory_keyword keyword,
+                 const char *keyword_as_string)
+{
+  directory_token_t *tok = find_opt_by_keyword(s, keyword);
+  if (PREDICT_UNLIKELY(!tok)) {
+    log_err(LD_BUG, "Missing %s [%d] in directory object that should have "
+         "been validated. Internal error.", keyword_as_string, (int)keyword);
+    tor_assert(tok);
+  }
+  return tok;
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
+ * NULL if no such keyword is found.
+ */
+directory_token_t *
+find_opt_by_keyword(const smartlist_t *s, directory_keyword keyword)
+{
+  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
+  return NULL;
+}
+
+/** If there are any directory_token_t entries in <b>s</b> whose keyword is
+ * <b>k</b>, return a newly allocated smartlist_t containing all such entries,
+ * in the same order in which they occur in <b>s</b>.  Otherwise return
+ * NULL. */
+smartlist_t *
+find_all_by_keyword(const smartlist_t *s, directory_keyword k)
+{
+  smartlist_t *out = NULL;
+  SMARTLIST_FOREACH(s, directory_token_t *, t,
+                    if (t->tp == k) {
+                    if (!out)
+                    out = smartlist_new();
+                    smartlist_add(out, t);
+                    });
+  return out;
+}
diff --git a/src/feature/nodelist/parsecommon.h b/src/feature/nodelist/parsecommon.h
new file mode 100644
index 0000000000..d0f3810c0b
--- /dev/null
+++ b/src/feature/nodelist/parsecommon.h
@@ -0,0 +1,324 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.h
+ * \brief Header file for parsecommon.c
+ **/
+
+#ifndef TOR_PARSECOMMON_H
+#define TOR_PARSECOMMON_H
+
+#include <stddef.h>
+
+struct smartlist_t;
+struct crypto_pk_t;
+struct memarea_t;
+
+/** Enumeration of possible token types.  The ones starting with K_ correspond
+* to directory 'keywords'. A_ is for an annotation, R or C is related to
+* hidden services, ERR_ is an error in the tokenizing process, EOF_ is an
+* end-of-file marker, and NIL_ is used to encode not-a-token.
+*/
+typedef enum {
+  K_ACCEPT = 0,
+  K_ACCEPT6,
+  K_DIRECTORY_SIGNATURE,
+  K_RECOMMENDED_SOFTWARE,
+  K_REJECT,
+  K_REJECT6,
+  K_ROUTER,
+  K_SIGNED_DIRECTORY,
+  K_SIGNING_KEY,
+  K_ONION_KEY,
+  K_ONION_KEY_NTOR,
+  K_ROUTER_SIGNATURE,
+  K_PUBLISHED,
+  K_RUNNING_ROUTERS,
+  K_ROUTER_STATUS,
+  K_PLATFORM,
+  K_PROTO,
+  K_OPT,
+  K_BANDWIDTH,
+  K_CONTACT,
+  K_NETWORK_STATUS,
+  K_UPTIME,
+  K_DIR_SIGNING_KEY,
+  K_FAMILY,
+  K_FINGERPRINT,
+  K_HIBERNATING,
+  K_READ_HISTORY,
+  K_WRITE_HISTORY,
+  K_NETWORK_STATUS_VERSION,
+  K_DIR_SOURCE,
+  K_DIR_OPTIONS,
+  K_CLIENT_VERSIONS,
+  K_SERVER_VERSIONS,
+  K_RECOMMENDED_CLIENT_PROTOCOLS,
+  K_RECOMMENDED_RELAY_PROTOCOLS,
+  K_REQUIRED_CLIENT_PROTOCOLS,
+  K_REQUIRED_RELAY_PROTOCOLS,
+  K_OR_ADDRESS,
+  K_ID,
+  K_P,
+  K_P6,
+  K_R,
+  K_A,
+  K_S,
+  K_V,
+  K_W,
+  K_M,
+  K_EXTRA_INFO,
+  K_EXTRA_INFO_DIGEST,
+  K_CACHES_EXTRA_INFO,
+  K_HIDDEN_SERVICE_DIR,
+  K_ALLOW_SINGLE_HOP_EXITS,
+  K_IPV6_POLICY,
+  K_ROUTER_SIG_ED25519,
+  K_IDENTITY_ED25519,
+  K_MASTER_KEY_ED25519,
+  K_ONION_KEY_CROSSCERT,
+  K_NTOR_ONION_KEY_CROSSCERT,
+
+  K_DIRREQ_END,
+  K_DIRREQ_V2_IPS,
+  K_DIRREQ_V3_IPS,
+  K_DIRREQ_V2_REQS,
+  K_DIRREQ_V3_REQS,
+  K_DIRREQ_V2_SHARE,
+  K_DIRREQ_V3_SHARE,
+  K_DIRREQ_V2_RESP,
+  K_DIRREQ_V3_RESP,
+  K_DIRREQ_V2_DIR,
+  K_DIRREQ_V3_DIR,
+  K_DIRREQ_V2_TUN,
+  K_DIRREQ_V3_TUN,
+  K_ENTRY_END,
+  K_ENTRY_IPS,
+  K_CELL_END,
+  K_CELL_PROCESSED,
+  K_CELL_QUEUED,
+  K_CELL_TIME,
+  K_CELL_CIRCS,
+  K_EXIT_END,
+  K_EXIT_WRITTEN,
+  K_EXIT_READ,
+  K_EXIT_OPENED,
+
+  K_DIR_KEY_CERTIFICATE_VERSION,
+  K_DIR_IDENTITY_KEY,
+  K_DIR_KEY_PUBLISHED,
+  K_DIR_KEY_EXPIRES,
+  K_DIR_KEY_CERTIFICATION,
+  K_DIR_KEY_CROSSCERT,
+  K_DIR_ADDRESS,
+  K_DIR_TUNNELLED,
+
+  K_VOTE_STATUS,
+  K_VALID_AFTER,
+  K_FRESH_UNTIL,
+  K_VALID_UNTIL,
+  K_VOTING_DELAY,
+
+  K_KNOWN_FLAGS,
+  K_PARAMS,
+  K_BW_WEIGHTS,
+  K_VOTE_DIGEST,
+  K_CONSENSUS_DIGEST,
+  K_ADDITIONAL_DIGEST,
+  K_ADDITIONAL_SIGNATURE,
+  K_CONSENSUS_METHODS,
+  K_CONSENSUS_METHOD,
+  K_LEGACY_DIR_KEY,
+  K_DIRECTORY_FOOTER,
+  K_SIGNING_CERT_ED,
+  K_SR_FLAG,
+  K_COMMIT,
+  K_PREVIOUS_SRV,
+  K_CURRENT_SRV,
+  K_PACKAGE,
+
+  A_PURPOSE,
+  A_LAST_LISTED,
+  A_UNKNOWN_,
+
+  R_RENDEZVOUS_SERVICE_DESCRIPTOR,
+  R_VERSION,
+  R_PERMANENT_KEY,
+  R_SECRET_ID_PART,
+  R_PUBLICATION_TIME,
+  R_PROTOCOL_VERSIONS,
+  R_INTRODUCTION_POINTS,
+  R_SIGNATURE,
+
+  R_HS_DESCRIPTOR, /* From version 3, this MUST be generic to all future
+                      descriptor versions thus making it R_. */
+  R3_DESC_LIFETIME,
+  R3_DESC_SIGNING_CERT,
+  R3_REVISION_COUNTER,
+  R3_SUPERENCRYPTED,
+  R3_SIGNATURE,
+  R3_CREATE2_FORMATS,
+  R3_INTRO_AUTH_REQUIRED,
+  R3_SINGLE_ONION_SERVICE,
+  R3_INTRODUCTION_POINT,
+  R3_INTRO_ONION_KEY,
+  R3_INTRO_AUTH_KEY,
+  R3_INTRO_ENC_KEY,
+  R3_INTRO_ENC_KEY_CERT,
+  R3_INTRO_LEGACY_KEY,
+  R3_INTRO_LEGACY_KEY_CERT,
+  R3_DESC_AUTH_TYPE,
+  R3_DESC_AUTH_KEY,
+  R3_DESC_AUTH_CLIENT,
+  R3_ENCRYPTED,
+
+  R_IPO_IDENTIFIER,
+  R_IPO_IP_ADDRESS,
+  R_IPO_ONION_PORT,
+  R_IPO_ONION_KEY,
+  R_IPO_SERVICE_KEY,
+
+  C_CLIENT_NAME,
+  C_DESCRIPTOR_COOKIE,
+  C_CLIENT_KEY,
+
+  ERR_,
+  EOF_,
+  NIL_
+} directory_keyword;
+
+/** Structure to hold a single directory token.
+ *
+ * We parse a directory by breaking it into "tokens", each consisting
+ * of a keyword, a line full of arguments, and a binary object.  The
+ * arguments and object are both optional, depending on the keyword
+ * type.
+ *
+ * This structure is only allocated in memareas; do not allocate it on
+ * the heap, or token_clear() won't work.
+ */
+typedef struct directory_token_t {
+  directory_keyword tp;        /**< Type of the token. */
+  int n_args:30;               /**< Number of elements in args */
+  char **args;                 /**< Array of arguments from keyword line. */
+
+  char *object_type;           /**< -----BEGIN [object_type]-----*/
+  size_t object_size;          /**< Bytes in object_body */
+  char *object_body;           /**< Contents of object, base64-decoded. */
+
+  struct crypto_pk_t *key;     /**< For public keys only.  Heap-allocated. */
+
+  char *error;                 /**< For ERR_ tokens only. */
+} directory_token_t;
+
+/** We use a table of rules to decide how to parse each token type. */
+
+/** Rules for whether the keyword needs an object. */
+typedef enum {
+  NO_OBJ,        /**< No object, ever. */
+  NEED_OBJ,      /**< Object is required. */
+  NEED_SKEY_1024,/**< Object is required, and must be a 1024 bit private key */
+  NEED_KEY_1024, /**< Object is required, and must be a 1024 bit public key */
+  NEED_KEY,      /**< Object is required, and must be a public key. */
+  OBJ_OK,        /**< Object is optional. */
+} obj_syntax;
+
+#define AT_START 1
+#define AT_END 2
+
+#define TS_ANNOTATIONS_OK 1
+#define TS_NOCHECK 2
+#define TS_NO_NEW_ANNOTATIONS 4
+
+/**
+ * @name macros for defining token rules
+ *
+ * Helper macros to define token tables.  's' is a string, 't' is a
+ * directory_keyword, 'a' is a trio of argument multiplicities, and 'o' is an
+ * object syntax.
+ */
+/**@{*/
+
+/** Appears to indicate the end of a table. */
+#define END_OF_TABLE { NULL, NIL_, 0,0,0, NO_OBJ, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions: used for obsolete document types */
+#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions on multiplicity or location. */
+#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item that must appear exactly once */
+#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0, 0 }
+/** An item that must appear exactly once, at the start of the document */
+#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START, 0 }
+/** An item that must appear exactly once, at the end of the document */
+#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END, 0 }
+/** An item that must appear one or more times */
+#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0, 0 }
+/** An item that must appear no more than once */
+#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
+/** An annotation that must appear no more than once */
+#define A01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 1 }
+
+/** Argument multiplicity: any number of arguments. */
+#define ARGS        0,INT_MAX,0
+/** Argument multiplicity: no arguments. */
+#define NO_ARGS     0,0,0
+/** Argument multiplicity: concatenate all arguments. */
+#define CONCAT_ARGS 1,1,1
+/** Argument multiplicity: at least <b>n</b> arguments. */
+#define GE(n)       n,INT_MAX,0
+/** Argument multiplicity: exactly <b>n</b> arguments. */
+#define EQ(n)       n,n,0
+/**@}*/
+
+/** Determines the parsing rules for a single token type. */
+typedef struct token_rule_t {
+  /** The string value of the keyword identifying the type of item. */
+  const char *t;
+  /** The corresponding directory_keyword enum. */
+  directory_keyword v;
+  /** Minimum number of arguments for this item */
+  int min_args;
+  /** Maximum number of arguments for this item */
+  int max_args;
+  /** If true, we concatenate all arguments for this item into a single
+   * string. */
+  int concat_args;
+  /** Requirements on object syntax for this item. */
+  obj_syntax os;
+  /** Lowest number of times this item may appear in a document. */
+  int min_cnt;
+  /** Highest number of times this item may appear in a document. */
+  int max_cnt;
+  /** One or more of AT_START/AT_END to limit where the item may appear in a
+   * document. */
+  int pos;
+  /** True iff this token is an annotation. */
+  int is_annotation;
+} token_rule_t;
+
+void token_clear(directory_token_t *tok);
+
+int tokenize_string(struct memarea_t *area,
+                    const char *start, const char *end,
+                    struct smartlist_t *out,
+                    token_rule_t *table,
+                    int flags);
+directory_token_t *get_next_token(struct memarea_t *area,
+                                  const char **s,
+                                  const char *eos,
+                                  token_rule_t *table);
+
+directory_token_t *find_by_keyword_(struct smartlist_t *s,
+                                    directory_keyword keyword,
+                                    const char *keyword_str);
+
+#define find_by_keyword(s, keyword) \
+  find_by_keyword_((s), (keyword), #keyword)
+
+directory_token_t *find_opt_by_keyword(const struct smartlist_t *s,
+                                       directory_keyword keyword);
+struct smartlist_t * find_all_by_keyword(const struct smartlist_t *s,
+                                         directory_keyword k);
+
+#endif /* !defined(TOR_PARSECOMMON_H) */
diff --git a/src/feature/nodelist/routerinfo_st.h b/src/feature/nodelist/routerinfo_st.h
new file mode 100644
index 0000000000..89a7702b30
--- /dev/null
+++ b/src/feature/nodelist/routerinfo_st.h
@@ -0,0 +1,108 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ROUTERINFO_ST_H
+#define ROUTERINFO_ST_H
+
+#include "or/signed_descriptor_st.h"
+
+struct curve25519_public_key_t;
+
+/** Information about another onion router in the network. */
+struct routerinfo_t {
+  signed_descriptor_t cache_info;
+  char *nickname; /**< Human-readable OR name. */
+
+  uint32_t addr; /**< IPv4 address of OR, in host order. */
+  uint16_t or_port; /**< Port for TLS connections. */
+  uint16_t dir_port; /**< Port for HTTP directory connections. */
+
+  /** A router's IPv6 address, if it has one. */
+  /* XXXXX187 Actually these should probably be part of a list of addresses,
+   * not just a special case.  Use abstractions to access these; don't do it
+   * directly. */
+  tor_addr_t ipv6_addr;
+  uint16_t ipv6_orport;
+
+  crypto_pk_t *onion_pkey; /**< Public RSA key for onions. */
+  crypto_pk_t *identity_pkey;  /**< Public RSA key for signing. */
+  /** Public curve25519 key for onions */
+  struct curve25519_public_key_t *onion_curve25519_pkey;
+  /** What's the earliest expiration time on all the certs in this
+   * routerinfo? */
+  time_t cert_expiration_time;
+
+  char *platform; /**< What software/operating system is this OR using? */
+
+  char *protocol_list; /**< Encoded list of subprotocol versions supported
+                        * by this OR */
+
+  /* link info */
+  uint32_t bandwidthrate; /**< How many bytes does this OR add to its token
+                           * bucket per second? */
+  uint32_t bandwidthburst; /**< How large is this OR's token bucket? */
+  /** How many bytes/s is this router known to handle? */
+  uint32_t bandwidthcapacity;
+  smartlist_t *exit_policy; /**< What streams will this OR permit
+                             * to exit on IPv4?  NULL for 'reject *:*'. */
+  /** What streams will this OR permit to exit on IPv6?
+   * NULL for 'reject *:*' */
+  struct short_policy_t *ipv6_exit_policy;
+  long uptime; /**< How many seconds the router claims to have been up */
+  smartlist_t *declared_family; /**< Nicknames of router which this router
+                                 * claims are its family. */
+  char *contact_info; /**< Declared contact info for this router. */
+  unsigned int is_hibernating:1; /**< Whether the router claims to be
+                                  * hibernating */
+  unsigned int caches_extra_info:1; /**< Whether the router says it caches and
+                                     * serves extrainfo documents. */
+  unsigned int allow_single_hop_exits:1;  /**< Whether the router says
+                                           * it allows single hop exits. */
+
+  unsigned int wants_to_be_hs_dir:1; /**< True iff this router claims to be
+                                      * a hidden service directory. */
+  unsigned int policy_is_reject_star:1; /**< True iff the exit policy for this
+                                         * router rejects everything. */
+  /** True if, after we have added this router, we should re-launch
+   * tests for it. */
+  unsigned int needs_retest_if_added:1;
+
+  /** True iff this router included "tunnelled-dir-server" in its descriptor,
+   * implying it accepts tunnelled directory requests, or it advertised
+   * dir_port > 0. */
+  unsigned int supports_tunnelled_dir_requests:1;
+
+  /** Used during voting to indicate that we should not include an entry for
+   * this routerinfo. Used only during voting. */
+  unsigned int omit_from_vote:1;
+
+  /** Flags to summarize the protocol versions for this routerinfo_t. */
+  protover_summary_flags_t pv;
+
+/** Tor can use this router for general positions in circuits; we got it
+ * from a directory server as usual, or we're an authority and a server
+ * uploaded it. */
+#define ROUTER_PURPOSE_GENERAL 0
+/** Tor should avoid using this router for circuit-building: we got it
+ * from a controller.  If the controller wants to use it, it'll have to
+ * ask for it by identity. */
+#define ROUTER_PURPOSE_CONTROLLER 1
+/** Tor should use this router only for bridge positions in circuits: we got
+ * it via a directory request from the bridge itself, or a bridge
+ * authority. */
+#define ROUTER_PURPOSE_BRIDGE 2
+/** Tor should not use this router; it was marked in cached-descriptors with
+ * a purpose we didn't recognize. */
+#define ROUTER_PURPOSE_UNKNOWN 255
+
+  /** In what way did we find out about this router?  One of ROUTER_PURPOSE_*.
+   * Routers of different purposes are kept segregated and used for different
+   * things; see notes on ROUTER_PURPOSE_* macros above.
+   */
+  uint8_t purpose;
+};
+
+#endif
diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c
new file mode 100644
index 0000000000..76a236ff20
--- /dev/null
+++ b/src/feature/nodelist/routerlist.c
@@ -0,0 +1,5848 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerlist.c
+ * \brief Code to
+ * maintain and access the global list of routerinfos for known
+ * servers.
+ *
+ * A "routerinfo_t" object represents a single self-signed router
+ * descriptor, as generated by a Tor relay in order to tell the rest of
+ * the world about its keys, address, and capabilities.  An
+ * "extrainfo_t" object represents an adjunct "extra-info" object,
+ * certified by a corresponding router descriptor, reporting more
+ * information about the relay that nearly all users will not need.
+ *
+ * Most users will not use router descriptors for most relays.  Instead,
+ * they use the information in microdescriptors and in the consensus
+ * networkstatus.
+ *
+ * Right now, routerinfo_t objects are used in these ways:
+ *  <ul>
+ *   <li>By clients, in order to learn about bridge keys and capabilities.
+ *     (Bridges aren't listed in the consensus networkstatus, so they
+ *     can't have microdescriptors.)
+ *   <li>By relays, since relays want more information about other relays
+ *     than they can learn from microdescriptors. (TODO: Is this still true?)
+ *   <li>By authorities, which receive them and use them to generate the
+ *     consensus and the microdescriptors.
+ *   <li>By all directory caches, which download them in case somebody
+ *     else wants them.
+ *  </ul>
+ *
+ * Routerinfos are mostly created by parsing them from a string, in
+ * routerparse.c. We store them to disk on receiving them, and
+ * periodically discard the ones we don't need. On restarting, we
+ * re-read them from disk. (This also applies to extrainfo documents, if
+ * we are configured to fetch them.)
+ *
+ * In order to keep our list of routerinfos up-to-date, we periodically
+ * check whether there are any listed in the latest consensus (or in the
+ * votes from other authorities, if we are an authority) that we don't
+ * have.  (This also applies to extrainfo documents, if we are
+ * configured to fetch them.)
+ *
+ * Almost nothing in Tor should use a routerinfo_t to refer directly to
+ * a relay; instead, almost everything should use node_t (implemented in
+ * nodelist.c), which provides a common interface to routerinfo_t,
+ * routerstatus_t, and microdescriptor_t.
+ *
+ * <br>
+ *
+ * This module also has some of the functions used for choosing random
+ * nodes according to different rules and weights.  Historically, they
+ * were all in this module.  Now, they are spread across this module,
+ * nodelist.c, and networkstatus.c.  (TODO: Fix that.)
+ *
+ * <br>
+ *
+ * (For historical reasons) this module also contains code for handling
+ * the list of fallback directories, the list of directory authorities,
+ * and the list of authority certificates.
+ *
+ * For the directory authorities, we have a list containing the public
+ * identity key, and contact points, for each authority.  The
+ * authorities receive descriptors from relays, and publish consensuses,
+ * descriptors, and microdescriptors.  This list is pre-configured.
+ *
+ * Fallback directories are well-known, stable, but untrusted directory
+ * caches that clients which have not yet bootstrapped can use to get
+ * their first networkstatus consensus, in order to find out where the
+ * Tor network really is.  This list is pre-configured in
+ * fallback_dirs.inc.  Every authority also serves as a fallback.
+ *
+ * Both fallback directories and directory authorities are are
+ * represented by a dir_server_t.
+ *
+ * Authority certificates are signed with authority identity keys; they
+ * are used to authenticate shorter-term authority signing keys. We
+ * fetch them when we find a consensus or a vote that has been signed
+ * with a signing key we don't recognize.  We cache them on disk and
+ * load them on startup.  Authority operators generate them with the
+ * "tor-gencert" utility.
+ *
+ * TODO: Authority certificates should be a separate module.
+ *
+ * TODO: dir_server_t stuff should be in a separate module.
+ **/
+
+#define ROUTERLIST_PRIVATE
+#include "or/or.h"
+#include "lib/err/backtrace.h"
+#include "or/bridges.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "or/circuitstats.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/entrynodes.h"
+#include "or/fp_pair.h"
+#include "or/geoip.h"
+#include "or/hibernate.h"
+#include "or/main.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/reasons.h"
+#include "or/rendcommon.h"
+#include "or/rendservice.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+#include "lib/sandbox/sandbox.h"
+#include "or/torcert.h"
+#include "lib/math/fp.h"
+
+#include "or/dirauth/dirvote.h"
+#include "or/dirauth/mode.h"
+
+#include "or/authority_cert_st.h"
+#include "or/dir_connection_st.h"
+#include "or/dir_server_st.h"
+#include "or/document_signature_st.h"
+#include "or/extrainfo_st.h"
+#include "or/networkstatus_st.h"
+#include "or/networkstatus_voter_info_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/vote_routerstatus_st.h"
+
+#include "lib/crypt_ops/digestset.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+// #define DEBUG_ROUTERLIST
+
+/****************************************************************************/
+
+/* Typed wrappers for different digestmap types; used to avoid type
+ * confusion. */
+
+DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t)
+DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t)
+DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t)
+DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
+#define SDMAP_FOREACH(map, keyvar, valvar)                              \
+  DIGESTMAP_FOREACH(sdmap_to_digestmap(map), keyvar, signed_descriptor_t *, \
+                    valvar)
+#define RIMAP_FOREACH(map, keyvar, valvar) \
+  DIGESTMAP_FOREACH(rimap_to_digestmap(map), keyvar, routerinfo_t *, valvar)
+#define EIMAP_FOREACH(map, keyvar, valvar) \
+  DIGESTMAP_FOREACH(eimap_to_digestmap(map), keyvar, extrainfo_t *, valvar)
+#define DSMAP_FOREACH(map, keyvar, valvar) \
+  DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
+                    valvar)
+#define eimap_free(map, fn) MAP_FREE_AND_NULL(eimap, (map), (fn))
+#define rimap_free(map, fn) MAP_FREE_AND_NULL(rimap, (map), (fn))
+#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn))
+#define sdmap_free(map, fn) MAP_FREE_AND_NULL(sdmap, (map), (fn))
+
+/* Forward declaration for cert_list_t */
+typedef struct cert_list_t cert_list_t;
+
+/* static function prototypes */
+static int compute_weighted_bandwidths(const smartlist_t *sl,
+                                       bandwidth_weight_rule_t rule,
+                                       double **bandwidths_out,
+                                       double *total_bandwidth_out);
+static const routerstatus_t *router_pick_trusteddirserver_impl(
+                const smartlist_t *sourcelist, dirinfo_type_t auth,
+                int flags, int *n_busy_out);
+static const routerstatus_t *router_pick_dirserver_generic(
+                              smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags);
+static void mark_all_dirservers_up(smartlist_t *server_list);
+static int signed_desc_digest_is_recognized(signed_descriptor_t *desc);
+static const char *signed_descriptor_get_body_impl(
+                                              const signed_descriptor_t *desc,
+                                              int with_annotations);
+static void list_pending_downloads(digestmap_t *result,
+                                   digest256map_t *result256,
+                                   int purpose, const char *prefix);
+static void list_pending_fpsk_downloads(fp_pair_map_t *result);
+static void launch_dummy_descriptor_download_as_needed(time_t now,
+                                   const or_options_t *options);
+static void download_status_reset_by_sk_in_cl(cert_list_t *cl,
+                                              const char *digest);
+static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+                                                const char *digest,
+                                                time_t now);
+
+/****************************************************************************/
+
+/** Global list of a dir_server_t object for each directory
+ * authority. */
+static smartlist_t *trusted_dir_servers = NULL;
+/** Global list of dir_server_t objects for all directory authorities
+ * and all fallback directory servers. */
+static smartlist_t *fallback_dir_servers = NULL;
+
+/** List of certificates for a single authority, and download status for
+ * latest certificate.
+ */
+struct cert_list_t {
+  /*
+   * The keys of download status map are cert->signing_key_digest for pending
+   * downloads by (identity digest/signing key digest) pair; functions such
+   * as authority_cert_get_by_digest() already assume these are unique.
+   */
+  struct digest_ds_map_t *dl_status_map;
+  /* There is also a dlstatus for the download by identity key only */
+  download_status_t dl_status_by_id;
+  smartlist_t *certs;
+};
+/** Map from v3 identity key digest to cert_list_t. */
+static digestmap_t *trusted_dir_certs = NULL;
+/** True iff any key certificate in at least one member of
+ * <b>trusted_dir_certs</b> has changed since we last flushed the
+ * certificates to disk. */
+static int trusted_dir_servers_certs_changed = 0;
+
+/** Global list of all of the routers that we know about. */
+static routerlist_t *routerlist = NULL;
+
+/** List of strings for nicknames we've already warned about and that are
+ * still unknown / unavailable. */
+static smartlist_t *warned_nicknames = NULL;
+
+/** The last time we tried to download any routerdesc, or 0 for "never".  We
+ * use this to rate-limit download attempts when the number of routerdescs to
+ * download is low. */
+static time_t last_descriptor_download_attempted = 0;
+
+/** Return the number of directory authorities whose type matches some bit set
+ * in <b>type</b>  */
+int
+get_n_authorities(dirinfo_type_t type)
+{
+  int n = 0;
+  if (!trusted_dir_servers)
+    return 0;
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+                    if (ds->type & type)
+                      ++n);
+  return n;
+}
+
+/** Initialise schedule, want_authority, and increment_on in the download
+ * status dlstatus, then call download_status_reset() on it.
+ * It is safe to call this function or download_status_reset() multiple times
+ * on a new dlstatus. But it should *not* be called after a dlstatus has been
+ * used to count download attempts or failures. */
+static void
+download_status_cert_init(download_status_t *dlstatus)
+{
+  dlstatus->schedule = DL_SCHED_CONSENSUS;
+  dlstatus->want_authority = DL_WANT_ANY_DIRSERVER;
+  dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE;
+  dlstatus->last_backoff_position = 0;
+  dlstatus->last_delay_used = 0;
+
+  /* Use the new schedule to set next_attempt_at */
+  download_status_reset(dlstatus);
+}
+
+/** Reset the download status of a specified element in a dsmap */
+static void
+download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest)
+{
+  download_status_t *dlstatus = NULL;
+
+  tor_assert(cl);
+  tor_assert(digest);
+
+  /* Make sure we have a dsmap */
+  if (!(cl->dl_status_map)) {
+    cl->dl_status_map = dsmap_new();
+  }
+  /* Look for a download_status_t in the map with this digest */
+  dlstatus = dsmap_get(cl->dl_status_map, digest);
+  /* Got one? */
+  if (!dlstatus) {
+    /* Insert before we reset */
+    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+    dsmap_set(cl->dl_status_map, digest, dlstatus);
+    download_status_cert_init(dlstatus);
+  }
+  tor_assert(dlstatus);
+  /* Go ahead and reset it */
+  download_status_reset(dlstatus);
+}
+
+/**
+ * Return true if the download for this signing key digest in cl is ready
+ * to be re-attempted.
+ */
+static int
+download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+                                     const char *digest,
+                                     time_t now)
+{
+  int rv = 0;
+  download_status_t *dlstatus = NULL;
+
+  tor_assert(cl);
+  tor_assert(digest);
+
+  /* Make sure we have a dsmap */
+  if (!(cl->dl_status_map)) {
+    cl->dl_status_map = dsmap_new();
+  }
+  /* Look for a download_status_t in the map with this digest */
+  dlstatus = dsmap_get(cl->dl_status_map, digest);
+  /* Got one? */
+  if (dlstatus) {
+    /* Use download_status_is_ready() */
+    rv = download_status_is_ready(dlstatus, now);
+  } else {
+    /*
+     * If we don't know anything about it, return 1, since we haven't
+     * tried this one before.  We need to create a new entry here,
+     * too.
+     */
+    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+    download_status_cert_init(dlstatus);
+    dsmap_set(cl->dl_status_map, digest, dlstatus);
+    rv = 1;
+  }
+
+  return rv;
+}
+
+/** Helper: Return the cert_list_t for an authority whose authority ID is
+ * <b>id_digest</b>, allocating a new list if necessary. */
+static cert_list_t *
+get_cert_list(const char *id_digest)
+{
+  cert_list_t *cl;
+  if (!trusted_dir_certs)
+    trusted_dir_certs = digestmap_new();
+  cl = digestmap_get(trusted_dir_certs, id_digest);
+  if (!cl) {
+    cl = tor_malloc_zero(sizeof(cert_list_t));
+    download_status_cert_init(&cl->dl_status_by_id);
+    cl->certs = smartlist_new();
+    cl->dl_status_map = dsmap_new();
+    digestmap_set(trusted_dir_certs, id_digest, cl);
+  }
+  return cl;
+}
+
+/** Return a list of authority ID digests with potentially enumerable lists
+ * of download_status_t objects; used by controller GETINFO queries.
+ */
+
+MOCK_IMPL(smartlist_t *,
+list_authority_ids_with_downloads, (void))
+{
+  smartlist_t *ids = smartlist_new();
+  digestmap_iter_t *i;
+  const char *digest;
+  char *tmp;
+  void *cl;
+
+  if (trusted_dir_certs) {
+    for (i = digestmap_iter_init(trusted_dir_certs);
+         !(digestmap_iter_done(i));
+         i = digestmap_iter_next(trusted_dir_certs, i)) {
+      /*
+       * We always have at least dl_status_by_id to query, so no need to
+       * probe deeper than the existence of a cert_list_t.
+       */
+      digestmap_iter_get(i, &digest, &cl);
+      tmp = tor_malloc(DIGEST_LEN);
+      memcpy(tmp, digest, DIGEST_LEN);
+      smartlist_add(ids, tmp);
+    }
+  }
+  /* else definitely no downloads going since nothing even has a cert list */
+
+  return ids;
+}
+
+/** Given an authority ID digest, return a pointer to the default download
+ * status, or NULL if there is no such entry in trusted_dir_certs */
+
+MOCK_IMPL(download_status_t *,
+id_only_download_status_for_authority_id, (const char *digest))
+{
+  download_status_t *dl = NULL;
+  cert_list_t *cl;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, digest);
+    if (cl) {
+      dl = &(cl->dl_status_by_id);
+    }
+  }
+
+  return dl;
+}
+
+/** Given an authority ID digest, return a smartlist of signing key digests
+ * for which download_status_t is potentially queryable, or NULL if no such
+ * authority ID digest is known. */
+
+MOCK_IMPL(smartlist_t *,
+list_sk_digests_for_authority_id, (const char *digest))
+{
+  smartlist_t *sks = NULL;
+  cert_list_t *cl;
+  dsmap_iter_t *i;
+  const char *sk_digest;
+  char *tmp;
+  download_status_t *dl;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, digest);
+    if (cl) {
+      sks = smartlist_new();
+      if (cl->dl_status_map) {
+        for (i = dsmap_iter_init(cl->dl_status_map);
+             !(dsmap_iter_done(i));
+             i = dsmap_iter_next(cl->dl_status_map, i)) {
+          /* Pull the digest out and add it to the list */
+          dsmap_iter_get(i, &sk_digest, &dl);
+          tmp = tor_malloc(DIGEST_LEN);
+          memcpy(tmp, sk_digest, DIGEST_LEN);
+          smartlist_add(sks, tmp);
+        }
+      }
+    }
+  }
+
+  return sks;
+}
+
+/** Given an authority ID digest and a signing key digest, return the
+ * download_status_t or NULL if none exists. */
+
+MOCK_IMPL(download_status_t *,
+download_status_for_authority_id_and_sk,(const char *id_digest,
+                                         const char *sk_digest))
+{
+  download_status_t *dl = NULL;
+  cert_list_t *cl = NULL;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, id_digest);
+    if (cl && cl->dl_status_map) {
+      dl = dsmap_get(cl->dl_status_map, sk_digest);
+    }
+  }
+
+  return dl;
+}
+
+#define cert_list_free(val) \
+  FREE_AND_NULL(cert_list_t, cert_list_free_, (val))
+
+/** Release all space held by a cert_list_t */
+static void
+cert_list_free_(cert_list_t *cl)
+{
+  if (!cl)
+    return;
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+                    authority_cert_free(cert));
+  smartlist_free(cl->certs);
+  dsmap_free(cl->dl_status_map, tor_free_);
+  tor_free(cl);
+}
+
+/** Wrapper for cert_list_free so we can pass it to digestmap_free */
+static void
+cert_list_free_void(void *cl)
+{
+  cert_list_free_(cl);
+}
+
+/** Reload the cached v3 key certificates from the cached-certs file in
+ * the data directory. Return 0 on success, -1 on failure. */
+int
+trusted_dirs_reload_certs(void)
+{
+  char *filename;
+  char *contents;
+  int r;
+
+  filename = get_cachedir_fname("cached-certs");
+  contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  tor_free(filename);
+  if (!contents)
+    return 0;
+  r = trusted_dirs_load_certs_from_string(
+        contents,
+        TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL);
+  tor_free(contents);
+  return r;
+}
+
+/** Helper: return true iff we already have loaded the exact cert
+ * <b>cert</b>. */
+static inline int
+already_have_cert(authority_cert_t *cert)
+{
+  cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest);
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+  {
+    if (tor_memeq(c->cache_info.signed_descriptor_digest,
+                cert->cache_info.signed_descriptor_digest,
+                DIGEST_LEN))
+      return 1;
+  });
+  return 0;
+}
+
+/** Load a bunch of new key certificates from the string <b>contents</b>.  If
+ * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are
+ * from the cache, and we don't need to flush them to disk.  If we are a
+ * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF.
+ * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
+ * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST.  If <b>flush</b> is true, we
+ * need to flush any changed certificates to disk now.  Return 0 on success,
+ * -1 if any certs fail to parse.
+ *
+ * If source_dir is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved certificates from, so try it first to
+ * fetch any missing certificates.
+ */
+int
+trusted_dirs_load_certs_from_string(const char *contents, int source,
+                                    int flush, const char *source_dir)
+{
+  dir_server_t *ds;
+  const char *s, *eos;
+  int failure_code = 0;
+  int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE);
+  int added_trusted_cert = 0;
+
+  for (s = contents; *s; s = eos) {
+    authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
+    cert_list_t *cl;
+    if (!cert) {
+      failure_code = -1;
+      break;
+    }
+    ds = trusteddirserver_get_by_v3_auth_digest(
+                                       cert->cache_info.identity_digest);
+    log_debug(LD_DIR, "Parsed certificate for %s",
+              ds ? ds->nickname : "unknown authority");
+
+    if (already_have_cert(cert)) {
+      /* we already have this one. continue. */
+      log_info(LD_DIR, "Skipping %s certificate for %s that we "
+               "already have.",
+               from_store ? "cached" : "downloaded",
+               ds ? ds->nickname : "an old or new authority");
+
+      /*
+       * A duplicate on download should be treated as a failure, so we call
+       * authority_cert_dl_failed() to reset the download status to make sure
+       * we can't try again.  Since we've implemented the fp-sk mechanism
+       * to download certs by signing key, this should be much rarer than it
+       * was and is perhaps cause for concern.
+       */
+      if (!from_store) {
+        if (authdir_mode(get_options())) {
+          log_warn(LD_DIR,
+                   "Got a certificate for %s, but we already have it. "
+                   "Maybe they haven't updated it. Waiting for a while.",
+                   ds ? ds->nickname : "an old or new authority");
+        } else {
+          log_info(LD_DIR,
+                   "Got a certificate for %s, but we already have it. "
+                   "Maybe they haven't updated it. Waiting for a while.",
+                   ds ? ds->nickname : "an old or new authority");
+        }
+
+        /*
+         * This is where we care about the source; authority_cert_dl_failed()
+         * needs to know whether the download was by fp or (fp,sk) pair to
+         * twiddle the right bit in the download map.
+         */
+        if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) {
+          authority_cert_dl_failed(cert->cache_info.identity_digest,
+                                   NULL, 404);
+        } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) {
+          authority_cert_dl_failed(cert->cache_info.identity_digest,
+                                   cert->signing_key_digest, 404);
+        }
+      }
+
+      authority_cert_free(cert);
+      continue;
+    }
+
+    if (ds) {
+      added_trusted_cert = 1;
+      log_info(LD_DIR, "Adding %s certificate for directory authority %s with "
+               "signing key %s", from_store ? "cached" : "downloaded",
+               ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
+    } else {
+      int adding = we_want_to_fetch_unknown_auth_certs(get_options());
+      log_info(LD_DIR, "%s %s certificate for unrecognized directory "
+               "authority with signing key %s",
+               adding ? "Adding" : "Not adding",
+               from_store ? "cached" : "downloaded",
+               hex_str(cert->signing_key_digest,DIGEST_LEN));
+      if (!adding) {
+        authority_cert_free(cert);
+        continue;
+      }
+    }
+
+    cl = get_cert_list(cert->cache_info.identity_digest);
+    smartlist_add(cl->certs, cert);
+    if (ds && cert->cache_info.published_on > ds->addr_current_at) {
+      /* Check to see whether we should update our view of the authority's
+       * address. */
+      if (cert->addr && cert->dir_port &&
+          (ds->addr != cert->addr ||
+           ds->dir_port != cert->dir_port)) {
+        char *a = tor_dup_ip(cert->addr);
+        log_notice(LD_DIR, "Updating address for directory authority %s "
+                   "from %s:%d to %s:%d based on certificate.",
+                   ds->nickname, ds->address, (int)ds->dir_port,
+                   a, cert->dir_port);
+        tor_free(a);
+        ds->addr = cert->addr;
+        ds->dir_port = cert->dir_port;
+      }
+      ds->addr_current_at = cert->cache_info.published_on;
+    }
+
+    if (!from_store)
+      trusted_dir_servers_certs_changed = 1;
+  }
+
+  if (flush)
+    trusted_dirs_flush_certs_to_disk();
+
+  /* call this even if failure_code is <0, since some certs might have
+   * succeeded, but only pass source_dir if there were no failures,
+   * and at least one more authority certificate was added to the store.
+   * This avoids retrying a directory that's serving bad or entirely duplicate
+   * certificates. */
+  if (failure_code == 0 && added_trusted_cert) {
+    networkstatus_note_certs_arrived(source_dir);
+  } else {
+    networkstatus_note_certs_arrived(NULL);
+  }
+
+  return failure_code;
+}
+
+/** Save all v3 key certificates to the cached-certs file. */
+void
+trusted_dirs_flush_certs_to_disk(void)
+{
+  char *filename;
+  smartlist_t *chunks;
+
+  if (!trusted_dir_servers_certs_changed || !trusted_dir_certs)
+    return;
+
+  chunks = smartlist_new();
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+          {
+            sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t));
+            c->bytes = cert->cache_info.signed_descriptor_body;
+            c->len = cert->cache_info.signed_descriptor_len;
+            smartlist_add(chunks, c);
+          });
+  } DIGESTMAP_FOREACH_END;
+
+  filename = get_cachedir_fname("cached-certs");
+  if (write_chunks_to_file(filename, chunks, 0, 0)) {
+    log_warn(LD_FS, "Error writing certificates to disk.");
+  }
+  tor_free(filename);
+  SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c));
+  smartlist_free(chunks);
+
+  trusted_dir_servers_certs_changed = 0;
+}
+
+static int
+compare_certs_by_pubdates(const void **_a, const void **_b)
+{
+  const authority_cert_t *cert1 = *_a, *cert2=*_b;
+
+  if (cert1->cache_info.published_on < cert2->cache_info.published_on)
+    return -1;
+  else if (cert1->cache_info.published_on >  cert2->cache_info.published_on)
+    return 1;
+  else
+    return 0;
+}
+
+/** Remove all expired v3 authority certificates that have been superseded for
+ * more than 48 hours or, if not expired, that were published more than 7 days
+ * before being superseded. (If the most recent cert was published more than 48
+ * hours ago, then we aren't going to get any consensuses signed with older
+ * keys.) */
+static void
+trusted_dirs_remove_old_certs(void)
+{
+  time_t now = time(NULL);
+#define DEAD_CERT_LIFETIME (2*24*60*60)
+#define SUPERSEDED_CERT_LIFETIME (2*24*60*60)
+  if (!trusted_dir_certs)
+    return;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    /* Sort the list from first-published to last-published */
+    smartlist_sort(cl->certs, compare_certs_by_pubdates);
+
+    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+      if (cert_sl_idx == smartlist_len(cl->certs) - 1) {
+        /* This is the most recently published cert.  Keep it. */
+        continue;
+      }
+      authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1);
+      const time_t next_cert_published = next_cert->cache_info.published_on;
+      if (next_cert_published > now) {
+        /* All later certs are published in the future. Keep everything
+         * we didn't discard. */
+        break;
+      }
+      int should_remove = 0;
+      if (cert->expires + DEAD_CERT_LIFETIME < now) {
+        /* Certificate has been expired for at least DEAD_CERT_LIFETIME.
+         * Remove it. */
+        should_remove = 1;
+      } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) {
+        /* Certificate has been superseded for OLD_CERT_LIFETIME.
+         * Remove it.
+         */
+        should_remove = 1;
+      }
+      if (should_remove) {
+        SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert);
+        authority_cert_free(cert);
+        trusted_dir_servers_certs_changed = 1;
+      }
+    } SMARTLIST_FOREACH_END(cert);
+
+  } DIGESTMAP_FOREACH_END;
+#undef DEAD_CERT_LIFETIME
+#undef OLD_CERT_LIFETIME
+
+  trusted_dirs_flush_certs_to_disk();
+}
+
+/** Return the newest v3 authority certificate whose v3 authority identity key
+ * has digest <b>id_digest</b>.  Return NULL if no such authority is known,
+ * or it has no certificate. */
+authority_cert_t *
+authority_cert_get_newest_by_id(const char *id_digest)
+{
+  cert_list_t *cl;
+  authority_cert_t *best = NULL;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return NULL;
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+  {
+    if (!best || cert->cache_info.published_on > best->cache_info.published_on)
+      best = cert;
+  });
+  return best;
+}
+
+/** Return the newest v3 authority certificate whose directory signing key has
+ * digest <b>sk_digest</b>. Return NULL if no such certificate is known.
+ */
+authority_cert_t *
+authority_cert_get_by_sk_digest(const char *sk_digest)
+{
+  authority_cert_t *c;
+  if (!trusted_dir_certs)
+    return NULL;
+
+  if ((c = get_my_v3_authority_cert()) &&
+      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+    return c;
+  if ((c = get_my_v3_legacy_cert()) &&
+      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+    return c;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+    {
+      if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+        return cert;
+    });
+  } DIGESTMAP_FOREACH_END;
+  return NULL;
+}
+
+/** Return the v3 authority certificate with signing key matching
+ * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>.
+ * Return NULL if no such authority is known. */
+authority_cert_t *
+authority_cert_get_by_digests(const char *id_digest,
+                              const char *sk_digest)
+{
+  cert_list_t *cl;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return NULL;
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+    if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+      return cert; );
+
+  return NULL;
+}
+
+/** Add every known authority_cert_t to <b>certs_out</b>. */
+void
+authority_cert_get_all(smartlist_t *certs_out)
+{
+  tor_assert(certs_out);
+  if (!trusted_dir_certs)
+    return;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+                      smartlist_add(certs_out, c));
+  } DIGESTMAP_FOREACH_END;
+}
+
+/** Called when an attempt to download a certificate with the authority with
+ * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest
+ * fails with HTTP response code <b>status</b>: remember the failure, so we
+ * don't try again immediately. */
+void
+authority_cert_dl_failed(const char *id_digest,
+                         const char *signing_key_digest, int status)
+{
+  cert_list_t *cl;
+  download_status_t *dlstatus = NULL;
+  char id_digest_str[2*DIGEST_LEN+1];
+  char sk_digest_str[2*DIGEST_LEN+1];
+
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return;
+
+  /*
+   * Are we noting a failed download of the latest cert for the id digest,
+   * or of a download by (id, signing key) digest pair?
+   */
+  if (!signing_key_digest) {
+    /* Just by id digest */
+    download_status_failed(&cl->dl_status_by_id, status);
+  } else {
+    /* Reset by (id, signing key) digest pair
+     *
+     * Look for a download_status_t in the map with this digest
+     */
+    dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest);
+    /* Got one? */
+    if (dlstatus) {
+      download_status_failed(dlstatus, status);
+    } else {
+      /*
+       * Do this rather than hex_str(), since hex_str clobbers
+       * old results and we call twice in the param list.
+       */
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    id_digest, DIGEST_LEN);
+      base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                    signing_key_digest, DIGEST_LEN);
+      log_warn(LD_BUG,
+               "Got failure for cert fetch with (fp,sk) = (%s,%s), with "
+               "status %d, but knew nothing about the download.",
+               id_digest_str, sk_digest_str, status);
+    }
+  }
+}
+
+static const char *BAD_SIGNING_KEYS[] = {
+  "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31
+  "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26
+  "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09
+  "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09
+  "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56
+  "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01
+  "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30
+  "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09
+  "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10
+  "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57
+  NULL,
+};
+
+/** Return true iff <b>cert</b> authenticates some atuhority signing key
+ * which, because of the old openssl heartbleed vulnerability, should
+ * never be trusted. */
+int
+authority_cert_is_blacklisted(const authority_cert_t *cert)
+{
+  char hex_digest[HEX_DIGEST_LEN+1];
+  int i;
+  base16_encode(hex_digest, sizeof(hex_digest),
+                cert->signing_key_digest, sizeof(cert->signing_key_digest));
+
+  for (i = 0; BAD_SIGNING_KEYS[i]; ++i) {
+    if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+/** Return true iff when we've been getting enough failures when trying to
+ * download the certificate with ID digest <b>id_digest</b> that we're willing
+ * to start bugging the user about it. */
+int
+authority_cert_dl_looks_uncertain(const char *id_digest)
+{
+#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2
+  cert_list_t *cl;
+  int n_failures;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return 0;
+
+  n_failures = download_status_get_n_failures(&cl->dl_status_by_id);
+  return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER;
+}
+
+/* Fetch the authority certificates specified in resource.
+ * If we are a bridge client, and node is a configured bridge, fetch from node
+ * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from
+ * rs. Otherwise, fetch from a random directory mirror. */
+static void
+authority_certs_fetch_resource_impl(const char *resource,
+                                    const char *dir_hint,
+                                    const node_t *node,
+                                    const routerstatus_t *rs)
+{
+  const or_options_t *options = get_options();
+  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                                            resource);
+
+  /* Make sure bridge clients never connect to anything but a bridge */
+  if (options->UseBridges) {
+    if (node && !node_is_a_configured_bridge(node)) {
+      /* If we're using bridges, and node is not a bridge, use a 3-hop path. */
+      get_via_tor = 1;
+    } else if (!node) {
+      /* If we're using bridges, and there's no node, use a 3-hop path. */
+      get_via_tor = 1;
+    }
+  }
+
+  const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
+                                                    : DIRIND_ONEHOP;
+
+  directory_request_t *req = NULL;
+  /* If we've just downloaded a consensus from a bridge, re-use that
+   * bridge */
+  if (options->UseBridges && node && node->ri && !get_via_tor) {
+    /* clients always make OR connections to bridges */
+    tor_addr_port_t or_ap;
+    /* we are willing to use a non-preferred address if we need to */
+    fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
+                                         &or_ap);
+
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_or_addr_port(req, &or_ap);
+    if (dir_hint)
+      directory_request_set_directory_id_digest(req, dir_hint);
+  } else if (rs) {
+    /* And if we've just downloaded a consensus from a directory, re-use that
+     * directory */
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_routerstatus(req, rs);
+  }
+
+  if (req) {
+    /* We've set up a request object -- fill in the other request fields, and
+     * send the request.  */
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
+    return;
+  }
+
+  /* Otherwise, we want certs from a random fallback or directory
+   * mirror, because they will almost always succeed. */
+  directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                               resource, PDS_RETRY_IF_NO_SERVERS,
+                               DL_WANT_ANY_DIRSERVER);
+}
+
+/** Try to download any v3 authority certificates that we may be missing.  If
+ * <b>status</b> is provided, try to get all the ones that were used to sign
+ * <b>status</b>.  Additionally, try to have a non-expired certificate for
+ * every V3 authority in trusted_dir_servers.  Don't fetch certificates we
+ * already have.
+ *
+ * If dir_hint is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved a consensus or certificates from, so try
+ * it first to fetch any missing certificates.
+ **/
+void
+authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+                              const char *dir_hint)
+{
+  /*
+   * The pending_id digestmap tracks pending certificate downloads by
+   * identity digest; the pending_cert digestmap tracks pending downloads
+   * by (identity digest, signing key digest) pairs.
+   */
+  digestmap_t *pending_id;
+  fp_pair_map_t *pending_cert;
+  /*
+   * The missing_id_digests smartlist will hold a list of id digests
+   * we want to fetch the newest cert for; the missing_cert_digests
+   * smartlist will hold a list of fp_pair_t with an identity and
+   * signing key digest.
+   */
+  smartlist_t *missing_cert_digests, *missing_id_digests;
+  char *resource = NULL;
+  cert_list_t *cl;
+  const or_options_t *options = get_options();
+  const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options);
+  fp_pair_t *fp_tmp = NULL;
+  char id_digest_str[2*DIGEST_LEN+1];
+  char sk_digest_str[2*DIGEST_LEN+1];
+
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+
+  pending_cert = fp_pair_map_new();
+  pending_id = digestmap_new();
+  missing_cert_digests = smartlist_new();
+  missing_id_digests = smartlist_new();
+
+  /*
+   * First, we get the lists of already pending downloads so we don't
+   * duplicate effort.
+   */
+  list_pending_downloads(pending_id, NULL,
+                         DIR_PURPOSE_FETCH_CERTIFICATE, "fp/");
+  list_pending_fpsk_downloads(pending_cert);
+
+  /*
+   * Now, we download any trusted authority certs we don't have by
+   * identity digest only.  This gets the latest cert for that authority.
+   */
+  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds) {
+    int found = 0;
+    if (!(ds->type & V3_DIRINFO))
+      continue;
+    if (smartlist_contains_digest(missing_id_digests,
+                                  ds->v3_identity_digest))
+      continue;
+    cl = get_cert_list(ds->v3_identity_digest);
+    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+      if (now < cert->expires) {
+        /* It's not expired, and we weren't looking for something to
+         * verify a consensus with.  Call it done. */
+        download_status_reset(&(cl->dl_status_by_id));
+        /* No sense trying to download it specifically by signing key hash */
+        download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest);
+        found = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(cert);
+    if (!found &&
+        download_status_is_ready(&(cl->dl_status_by_id), now) &&
+        !digestmap_get(pending_id, ds->v3_identity_digest)) {
+      log_info(LD_DIR,
+               "No current certificate known for authority %s "
+               "(ID digest %s); launching request.",
+               ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN));
+      smartlist_add(missing_id_digests, ds->v3_identity_digest);
+    }
+  } SMARTLIST_FOREACH_END(ds);
+
+  /*
+   * Next, if we have a consensus, scan through it and look for anything
+   * signed with a key from a cert we don't have.  Those get downloaded
+   * by (fp,sk) pair, but if we don't know any certs at all for the fp
+   * (identity digest), and it's one of the trusted dir server certs
+   * we started off above or a pending download in pending_id, don't
+   * try to get it yet.  Most likely, the one we'll get for that will
+   * have the right signing key too, and we'd just be downloading
+   * redundantly.
+   */
+  if (status) {
+    SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *,
+                            voter) {
+      if (!smartlist_len(voter->sigs))
+        continue; /* This authority never signed this consensus, so don't
+                   * go looking for a cert with key digest 0000000000. */
+      if (!keep_unknown &&
+          !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
+        continue; /* We don't want unknown certs, and we don't know this
+                   * authority.*/
+
+      /*
+       * If we don't know *any* cert for this authority, and a download by ID
+       * is pending or we added it to missing_id_digests above, skip this
+       * one for now to avoid duplicate downloads.
+       */
+      cl = get_cert_list(voter->identity_digest);
+      if (smartlist_len(cl->certs) == 0) {
+        /* We have no certs at all for this one */
+
+        /* Do we have a download of one pending? */
+        if (digestmap_get(pending_id, voter->identity_digest))
+          continue;
+
+        /*
+         * Are we about to launch a download of one due to the trusted
+         * dir server check above?
+         */
+        if (smartlist_contains_digest(missing_id_digests,
+                                      voter->identity_digest))
+          continue;
+      }
+
+      SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
+        authority_cert_t *cert =
+          authority_cert_get_by_digests(voter->identity_digest,
+                                        sig->signing_key_digest);
+        if (cert) {
+          if (now < cert->expires)
+            download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest);
+          continue;
+        }
+        if (download_status_is_ready_by_sk_in_cl(
+              cl, sig->signing_key_digest, now) &&
+            !fp_pair_map_get_by_digests(pending_cert,
+                                        voter->identity_digest,
+                                        sig->signing_key_digest)) {
+          /*
+           * Do this rather than hex_str(), since hex_str clobbers
+           * old results and we call twice in the param list.
+           */
+          base16_encode(id_digest_str, sizeof(id_digest_str),
+                        voter->identity_digest, DIGEST_LEN);
+          base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                        sig->signing_key_digest, DIGEST_LEN);
+
+          if (voter->nickname) {
+            log_info(LD_DIR,
+                     "We're missing a certificate from authority %s "
+                     "(ID digest %s) with signing key %s: "
+                     "launching request.",
+                     voter->nickname, id_digest_str, sk_digest_str);
+          } else {
+            log_info(LD_DIR,
+                     "We're missing a certificate from authority ID digest "
+                     "%s with signing key %s: launching request.",
+                     id_digest_str, sk_digest_str);
+          }
+
+          /* Allocate a new fp_pair_t to append */
+          fp_tmp = tor_malloc(sizeof(*fp_tmp));
+          memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first));
+          memcpy(fp_tmp->second, sig->signing_key_digest,
+                 sizeof(fp_tmp->second));
+          smartlist_add(missing_cert_digests, fp_tmp);
+        }
+      } SMARTLIST_FOREACH_END(sig);
+    } SMARTLIST_FOREACH_END(voter);
+  }
+
+  /* Bridge clients look up the node for the dir_hint */
+  const node_t *node = NULL;
+  /* All clients, including bridge clients, look up the routerstatus for the
+   * dir_hint */
+  const routerstatus_t *rs = NULL;
+
+  /* If we still need certificates, try the directory that just successfully
+   * served us a consensus or certificates.
+   * As soon as the directory fails to provide additional certificates, we try
+   * another, randomly selected directory. This avoids continual retries.
+   * (We only ever have one outstanding request per certificate.)
+   */
+  if (dir_hint) {
+    if (options->UseBridges) {
+      /* Bridge clients try the nodelist. If the dir_hint is from an authority,
+       * or something else fetched over tor, we won't find the node here, but
+       * we will find the rs. */
+      node = node_get_by_id(dir_hint);
+    }
+
+    /* All clients try the consensus routerstatus, then the fallback
+     * routerstatus */
+    rs = router_get_consensus_status_by_id(dir_hint);
+    if (!rs) {
+      /* This will also find authorities */
+      const dir_server_t *ds = router_get_fallback_dirserver_by_digest(
+                                                                    dir_hint);
+      if (ds) {
+        rs = &ds->fake_status;
+      }
+    }
+
+    if (!node && !rs) {
+      log_warn(LD_BUG, "Directory %s delivered a consensus, but %s"
+               "no routerstatus could be found for it.",
+               options->UseBridges ? "no node and " : "",
+               hex_str(dir_hint, DIGEST_LEN));
+    }
+  }
+
+  /* Do downloads by identity digest */
+  if (smartlist_len(missing_id_digests) > 0) {
+    int need_plus = 0;
+    smartlist_t *fps = smartlist_new();
+
+    smartlist_add_strdup(fps, "fp/");
+
+    SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
+      char *fp = NULL;
+
+      if (digestmap_get(pending_id, d))
+        continue;
+
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    d, DIGEST_LEN);
+
+      if (need_plus) {
+        tor_asprintf(&fp, "+%s", id_digest_str);
+      } else {
+        /* No need for tor_asprintf() in this case; first one gets no '+' */
+        fp = tor_strdup(id_digest_str);
+        need_plus = 1;
+      }
+
+      smartlist_add(fps, fp);
+    } SMARTLIST_FOREACH_END(d);
+
+    if (smartlist_len(fps) > 1) {
+      resource = smartlist_join_strings(fps, "", 0, NULL);
+      /* node and rs are directories that just gave us a consensus or
+       * certificates */
+      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+      tor_free(resource);
+    }
+    /* else we didn't add any: they were all pending */
+
+    SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp));
+    smartlist_free(fps);
+  }
+
+  /* Do downloads by identity digest/signing key pair */
+  if (smartlist_len(missing_cert_digests) > 0) {
+    int need_plus = 0;
+    smartlist_t *fp_pairs = smartlist_new();
+
+    smartlist_add_strdup(fp_pairs, "fp-sk/");
+
+    SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
+      char *fp_pair = NULL;
+
+      if (fp_pair_map_get(pending_cert, d))
+        continue;
+
+      /* Construct string encodings of the digests */
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    d->first, DIGEST_LEN);
+      base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                    d->second, DIGEST_LEN);
+
+      /* Now tor_asprintf() */
+      if (need_plus) {
+        tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str);
+      } else {
+        /* First one in the list doesn't get a '+' */
+        tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str);
+        need_plus = 1;
+      }
+
+      /* Add it to the list of pairs to request */
+      smartlist_add(fp_pairs, fp_pair);
+    } SMARTLIST_FOREACH_END(d);
+
+    if (smartlist_len(fp_pairs) > 1) {
+      resource = smartlist_join_strings(fp_pairs, "", 0, NULL);
+      /* node and rs are directories that just gave us a consensus or
+       * certificates */
+      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+      tor_free(resource);
+    }
+    /* else they were all pending */
+
+    SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p));
+    smartlist_free(fp_pairs);
+  }
+
+  smartlist_free(missing_id_digests);
+  SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p));
+  smartlist_free(missing_cert_digests);
+  digestmap_free(pending_id, NULL);
+  fp_pair_map_free(pending_cert, NULL);
+}
+
+/* Router descriptor storage.
+ *
+ * Routerdescs are stored in a big file, named "cached-descriptors".  As new
+ * routerdescs arrive, we append them to a journal file named
+ * "cached-descriptors.new".
+ *
+ * From time to time, we replace "cached-descriptors" with a new file
+ * containing only the live, non-superseded descriptors, and clear
+ * cached-routers.new.
+ *
+ * On startup, we read both files.
+ */
+
+/** Helper: return 1 iff the router log is so big we want to rebuild the
+ * store. */
+static int
+router_should_rebuild_store(desc_store_t *store)
+{
+  if (store->store_len > (1<<16))
+    return (store->journal_len > store->store_len / 2 ||
+            store->bytes_dropped > store->store_len / 2);
+  else
+    return store->journal_len > (1<<15);
+}
+
+/** Return the desc_store_t in <b>rl</b> that should be used to store
+ * <b>sd</b>. */
+static inline desc_store_t *
+desc_get_store(routerlist_t *rl, const signed_descriptor_t *sd)
+{
+  if (sd->is_extrainfo)
+    return &rl->extrainfo_store;
+  else
+    return &rl->desc_store;
+}
+
+/** Add the signed_descriptor_t in <b>desc</b> to the router
+ * journal; change its saved_location to SAVED_IN_JOURNAL and set its
+ * offset appropriately. */
+static int
+signed_desc_append_to_journal(signed_descriptor_t *desc,
+                              desc_store_t *store)
+{
+  char *fname = get_cachedir_fname_suffix(store->fname_base, ".new");
+  const char *body = signed_descriptor_get_body_impl(desc,1);
+  size_t len = desc->signed_descriptor_len + desc->annotations_len;
+
+  if (append_bytes_to_file(fname, body, len, 1)) {
+    log_warn(LD_FS, "Unable to store router descriptor");
+    tor_free(fname);
+    return -1;
+  }
+  desc->saved_location = SAVED_IN_JOURNAL;
+  tor_free(fname);
+
+  desc->saved_offset = store->journal_len;
+  store->journal_len += len;
+
+  return 0;
+}
+
+/** Sorting helper: return &lt;0, 0, or &gt;0 depending on whether the
+ * signed_descriptor_t* in *<b>a</b> is older, the same age as, or newer than
+ * the signed_descriptor_t* in *<b>b</b>. */
+static int
+compare_signed_descriptors_by_age_(const void **_a, const void **_b)
+{
+  const signed_descriptor_t *r1 = *_a, *r2 = *_b;
+  return (int)(r1->published_on - r2->published_on);
+}
+
+#define RRS_FORCE 1
+#define RRS_DONT_REMOVE_OLD 2
+
+/** If the journal of <b>store</b> is too long, or if RRS_FORCE is set in
+ * <b>flags</b>, then atomically replace the saved router store with the
+ * routers currently in our routerlist, and clear the journal.  Unless
+ * RRS_DONT_REMOVE_OLD is set in <b>flags</b>, delete expired routers before
+ * rebuilding the store.  Return 0 on success, -1 on failure.
+ */
+static int
+router_rebuild_store(int flags, desc_store_t *store)
+{
+  smartlist_t *chunk_list = NULL;
+  char *fname = NULL, *fname_tmp = NULL;
+  int r = -1;
+  off_t offset = 0;
+  smartlist_t *signed_descriptors = NULL;
+  int nocache=0;
+  size_t total_expected_len = 0;
+  int had_any;
+  int force = flags & RRS_FORCE;
+
+  if (!force && !router_should_rebuild_store(store)) {
+    r = 0;
+    goto done;
+  }
+  if (!routerlist) {
+    r = 0;
+    goto done;
+  }
+
+  if (store->type == EXTRAINFO_STORE)
+    had_any = !eimap_isempty(routerlist->extra_info_map);
+  else
+    had_any = (smartlist_len(routerlist->routers)+
+               smartlist_len(routerlist->old_routers))>0;
+
+  /* Don't save deadweight. */
+  if (!(flags & RRS_DONT_REMOVE_OLD))
+    routerlist_remove_old_routers();
+
+  log_info(LD_DIR, "Rebuilding %s cache", store->description);
+
+  fname = get_cachedir_fname(store->fname_base);
+  fname_tmp = get_cachedir_fname_suffix(store->fname_base, ".tmp");
+
+  chunk_list = smartlist_new();
+
+  /* We sort the routers by age to enhance locality on disk. */
+  signed_descriptors = smartlist_new();
+  if (store->type == EXTRAINFO_STORE) {
+    eimap_iter_t *iter;
+    for (iter = eimap_iter_init(routerlist->extra_info_map);
+         !eimap_iter_done(iter);
+         iter = eimap_iter_next(routerlist->extra_info_map, iter)) {
+      const char *key;
+      extrainfo_t *ei;
+      eimap_iter_get(iter, &key, &ei);
+      smartlist_add(signed_descriptors, &ei->cache_info);
+    }
+  } else {
+    SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd,
+                      smartlist_add(signed_descriptors, sd));
+    SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri,
+                      smartlist_add(signed_descriptors, &ri->cache_info));
+  }
+
+  smartlist_sort(signed_descriptors, compare_signed_descriptors_by_age_);
+
+  /* Now, add the appropriate members to chunk_list */
+  SMARTLIST_FOREACH_BEGIN(signed_descriptors, signed_descriptor_t *, sd) {
+      sized_chunk_t *c;
+      const char *body = signed_descriptor_get_body_impl(sd, 1);
+      if (!body) {
+        log_warn(LD_BUG, "No descriptor available for router.");
+        goto done;
+      }
+      if (sd->do_not_cache) {
+        ++nocache;
+        continue;
+      }
+      c = tor_malloc(sizeof(sized_chunk_t));
+      c->bytes = body;
+      c->len = sd->signed_descriptor_len + sd->annotations_len;
+      total_expected_len += c->len;
+      smartlist_add(chunk_list, c);
+  } SMARTLIST_FOREACH_END(sd);
+
+  if (write_chunks_to_file(fname_tmp, chunk_list, 1, 1)<0) {
+    log_warn(LD_FS, "Error writing router store to disk.");
+    goto done;
+  }
+
+  /* Our mmap is now invalid. */
+  if (store->mmap) {
+    int res = tor_munmap_file(store->mmap);
+    store->mmap = NULL;
+    if (res != 0) {
+      log_warn(LD_FS, "Unable to munmap route store in %s", fname);
+    }
+  }
+
+  if (replace_file(fname_tmp, fname)<0) {
+    log_warn(LD_FS, "Error replacing old router store: %s", strerror(errno));
+    goto done;
+  }
+
+  errno = 0;
+  store->mmap = tor_mmap_file(fname);
+  if (! store->mmap) {
+    if (errno == ERANGE) {
+      /* empty store.*/
+      if (total_expected_len) {
+        log_warn(LD_FS, "We wrote some bytes to a new descriptor file at '%s',"
+                 " but when we went to mmap it, it was empty!", fname);
+      } else if (had_any) {
+        log_info(LD_FS, "We just removed every descriptor in '%s'.  This is "
+                 "okay if we're just starting up after a long time. "
+                 "Otherwise, it's a bug.", fname);
+      }
+    } else {
+      log_warn(LD_FS, "Unable to mmap new descriptor file at '%s'.",fname);
+    }
+  }
+
+  log_info(LD_DIR, "Reconstructing pointers into cache");
+
+  offset = 0;
+  SMARTLIST_FOREACH_BEGIN(signed_descriptors, signed_descriptor_t *, sd) {
+      if (sd->do_not_cache)
+        continue;
+      sd->saved_location = SAVED_IN_CACHE;
+      if (store->mmap) {
+        tor_free(sd->signed_descriptor_body); // sets it to null
+        sd->saved_offset = offset;
+      }
+      offset += sd->signed_descriptor_len + sd->annotations_len;
+      signed_descriptor_get_body(sd); /* reconstruct and assert */
+  } SMARTLIST_FOREACH_END(sd);
+
+  tor_free(fname);
+  fname = get_cachedir_fname_suffix(store->fname_base, ".new");
+  write_str_to_file(fname, "", 1);
+
+  r = 0;
+  store->store_len = (size_t) offset;
+  store->journal_len = 0;
+  store->bytes_dropped = 0;
+ done:
+  smartlist_free(signed_descriptors);
+  tor_free(fname);
+  tor_free(fname_tmp);
+  if (chunk_list) {
+    SMARTLIST_FOREACH(chunk_list, sized_chunk_t *, c, tor_free(c));
+    smartlist_free(chunk_list);
+  }
+
+  return r;
+}
+
+/** Helper: Reload a cache file and its associated journal, setting metadata
+ * appropriately.  If <b>extrainfo</b> is true, reload the extrainfo store;
+ * else reload the router descriptor store. */
+static int
+router_reload_router_list_impl(desc_store_t *store)
+{
+  char *fname = NULL, *contents = NULL;
+  struct stat st;
+  int extrainfo = (store->type == EXTRAINFO_STORE);
+  store->journal_len = store->store_len = 0;
+
+  fname = get_cachedir_fname(store->fname_base);
+
+  if (store->mmap) {
+    /* get rid of it first */
+    int res = tor_munmap_file(store->mmap);
+    store->mmap = NULL;
+    if (res != 0) {
+      log_warn(LD_FS, "Failed to munmap %s", fname);
+      tor_free(fname);
+      return -1;
+    }
+  }
+
+  store->mmap = tor_mmap_file(fname);
+  if (store->mmap) {
+    store->store_len = store->mmap->size;
+    if (extrainfo)
+      router_load_extrainfo_from_string(store->mmap->data,
+                                        store->mmap->data+store->mmap->size,
+                                        SAVED_IN_CACHE, NULL, 0);
+    else
+      router_load_routers_from_string(store->mmap->data,
+                                      store->mmap->data+store->mmap->size,
+                                      SAVED_IN_CACHE, NULL, 0, NULL);
+  }
+
+  tor_free(fname);
+  fname = get_cachedir_fname_suffix(store->fname_base, ".new");
+  /* don't load empty files - we wouldn't get any data, even if we tried */
+  if (file_status(fname) == FN_FILE)
+    contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st);
+  if (contents) {
+    if (extrainfo)
+      router_load_extrainfo_from_string(contents, NULL,SAVED_IN_JOURNAL,
+                                        NULL, 0);
+    else
+      router_load_routers_from_string(contents, NULL, SAVED_IN_JOURNAL,
+                                      NULL, 0, NULL);
+    store->journal_len = (size_t) st.st_size;
+    tor_free(contents);
+  }
+
+  tor_free(fname);
+
+  if (store->journal_len) {
+    /* Always clear the journal on startup.*/
+    router_rebuild_store(RRS_FORCE, store);
+  } else if (!extrainfo) {
+    /* Don't cache expired routers. (This is in an else because
+     * router_rebuild_store() also calls remove_old_routers().) */
+    routerlist_remove_old_routers();
+  }
+
+  return 0;
+}
+
+/** Load all cached router descriptors and extra-info documents from the
+ * store. Return 0 on success and -1 on failure.
+ */
+int
+router_reload_router_list(void)
+{
+  routerlist_t *rl = router_get_routerlist();
+  if (router_reload_router_list_impl(&rl->desc_store))
+    return -1;
+  if (router_reload_router_list_impl(&rl->extrainfo_store))
+    return -1;
+  return 0;
+}
+
+/** Return a smartlist containing a list of dir_server_t * for all
+ * known trusted dirservers.  Callers must not modify the list or its
+ * contents.
+ */
+const smartlist_t *
+router_get_trusted_dir_servers(void)
+{
+  if (!trusted_dir_servers)
+    trusted_dir_servers = smartlist_new();
+
+  return trusted_dir_servers;
+}
+
+const smartlist_t *
+router_get_fallback_dir_servers(void)
+{
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  return fallback_dir_servers;
+}
+
+/** Try to find a running dirserver that supports operations of <b>type</b>.
+ *
+ * If there are no running dirservers in our routerlist and the
+ * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
+ * (including authorities) as running again, and pick one.
+ *
+ * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
+ * dirservers that we can't reach.
+ *
+ * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
+ * (if we're a dirserver).
+ *
+ * Don't pick a fallback directory mirror if any non-fallback is viable;
+ * (the fallback directory mirrors include the authorities)
+ * try to avoid using servers that have returned 503 recently.
+ */
+const routerstatus_t *
+router_pick_directory_server(dirinfo_type_t type, int flags)
+{
+  int busy = 0;
+  const routerstatus_t *choice;
+
+  if (!routerlist)
+    return NULL;
+
+  choice = router_pick_directory_server_impl(type, flags, &busy);
+  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+    return choice;
+
+  if (busy) {
+    /* If the reason that we got no server is that servers are "busy",
+     * we must be excluding good servers because we already have serverdesc
+     * fetches with them.  Do not mark down servers up because of this. */
+    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH)));
+    return NULL;
+  }
+
+  log_info(LD_DIR,
+           "No reachable router entries for dirservers. "
+           "Trying them all again.");
+  /* mark all fallback directory mirrors as up again */
+  mark_all_dirservers_up(fallback_dir_servers);
+  /* try again */
+  choice = router_pick_directory_server_impl(type, flags, NULL);
+  return choice;
+}
+
+/** Return the dir_server_t for the directory authority whose identity
+ * key hashes to <b>digest</b>, or NULL if no such authority is known.
+ */
+dir_server_t *
+router_get_trusteddirserver_by_digest(const char *digest)
+{
+  if (!trusted_dir_servers)
+    return NULL;
+
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Return the dir_server_t for the fallback dirserver whose identity
+ * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of
+ * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir
+ * and UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+dir_server_t *
+router_get_fallback_dirserver_by_digest(const char *digest)
+{
+  if (!fallback_dir_servers)
+    return NULL;
+
+  if (!digest)
+    return NULL;
+
+  SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>,
+ * or 0 if no such fallback is in the list of fallback_dir_servers.
+ * (fallback_dir_servers is affected by the FallbackDir and
+ * UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+int
+router_digest_is_fallback_dir(const char *digest)
+{
+  return (router_get_fallback_dirserver_by_digest(digest) != NULL);
+}
+
+/** Return the dir_server_t for the directory authority whose
+ * v3 identity key hashes to <b>digest</b>, or NULL if no such authority
+ * is known.
+ */
+MOCK_IMPL(dir_server_t *,
+trusteddirserver_get_by_v3_auth_digest, (const char *digest))
+{
+  if (!trusted_dir_servers)
+    return NULL;
+
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) &&
+           (ds->type & V3_DIRINFO))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Try to find a running directory authority. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_trusteddirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(trusted_dir_servers, type, flags);
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_fallback_dirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(fallback_dir_servers, type, flags);
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+static const routerstatus_t *
+router_pick_dirserver_generic(smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags)
+{
+  const routerstatus_t *choice;
+  int busy = 0;
+
+  choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy);
+  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+    return choice;
+  if (busy) {
+    /* If the reason that we got no server is that servers are "busy",
+     * we must be excluding good servers because we already have serverdesc
+     * fetches with them.  Do not mark down servers up because of this. */
+    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH)));
+    return NULL;
+  }
+
+  log_info(LD_DIR,
+           "No dirservers are reachable. Trying them all again.");
+  mark_all_dirservers_up(sourcelist);
+  return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL);
+}
+
+/* Check if we already have a directory fetch from ap, for serverdesc
+ * (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0.
+ */
+STATIC int
+router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc,
+                               int microdesc)
+{
+  if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) {
+    return 0;
+  }
+
+  /* XX/teor - we're not checking tunnel connections here, see #17848
+   */
+  if (serverdesc && (
+     connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC)
+  || connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) {
+    return 1;
+  }
+
+  if (microdesc && (
+     connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/* Check if we already have a directory fetch from the ipv4 or ipv6
+ * router, for serverdesc (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ */
+static int
+router_is_already_dir_fetching_(uint32_t ipv4_addr,
+                                const tor_addr_t *ipv6_addr,
+                                uint16_t dir_port,
+                                int serverdesc,
+                                int microdesc)
+{
+  tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
+
+  /* Assume IPv6 DirPort is the same as IPv4 DirPort */
+  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr);
+  ipv4_dir_ap.port = dir_port;
+  tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr);
+  ipv6_dir_ap.port = dir_port;
+
+  return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
+       || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
+}
+
+#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0
+#endif
+
+/* Log a message if rs is not found or not a preferred address */
+static void
+router_picked_poor_directory_log(const routerstatus_t *rs)
+{
+  const networkstatus_t *usable_consensus;
+  usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL),
+                                                 usable_consensus_flavor());
+
+#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+  /* Don't log early in the bootstrap process, it's normal to pick from a
+   * small pool of nodes. Of course, this won't help if we're trying to
+   * diagnose bootstrap issues. */
+  if (!smartlist_len(nodelist_get_list()) || !usable_consensus
+      || !router_have_minimum_dir_info()) {
+    return;
+  }
+#endif /* !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP */
+
+  /* We couldn't find a node, or the one we have doesn't fit our preferences.
+   * Sometimes this is normal, sometimes it can be a reachability issue. */
+  if (!rs) {
+    /* This happens a lot, so it's at debug level */
+    log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but "
+              "we couldn't find a directory that fit our criteria. "
+              "Perhaps we will succeed next time with less strict criteria.");
+  } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1)
+             && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1)
+             ) {
+    /* This is rare, and might be interesting to users trying to diagnose
+     * connection issues on dual-stack machines. */
+    log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir "
+             "addresses for launching an outgoing connection: "
+             "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d",
+             routerstatus_describe(rs),
+             fmt_addr32(rs->addr), rs->or_port,
+             rs->dir_port, fmt_addr(&rs->ipv6_addr),
+             rs->ipv6_orport, rs->dir_port);
+  }
+}
+
+#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+
+/** How long do we avoid using a directory server after it's given us a 503? */
+#define DIR_503_TIMEOUT (60*60)
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version.
+ * Must be called before RETRY_WITHOUT_EXCLUDE().
+ *
+ * If we got no result, and we are applying IP preferences, and we are a
+ * client that could use an alternate IP version, try again with the
+ * opposite preferences. */
+#define RETRY_ALTERNATE_IP_VERSION(retry_label)                               \
+  STMT_BEGIN                                                                  \
+    if (result == NULL && try_ip_pref && options->ClientUseIPv4               \
+        && fascist_firewall_use_ipv6(options) && !server_mode(options)        \
+        && !n_busy) {                                                         \
+      n_excluded = 0;                                                         \
+      n_busy = 0;                                                             \
+      try_ip_pref = 0;                                                        \
+      goto retry_label;                                                       \
+    }                                                                         \
+  STMT_END                                                                    \
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with
+ * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION().
+ *
+ * If we got no result, and we are excluding nodes, and StrictNodes is
+ * not set, try again without excluding nodes. */
+#define RETRY_WITHOUT_EXCLUDE(retry_label)                                    \
+  STMT_BEGIN                                                                  \
+    if (result == NULL && try_excluding && !options->StrictNodes              \
+        && n_excluded && !n_busy) {                                           \
+      try_excluding = 0;                                                      \
+      n_excluded = 0;                                                         \
+      n_busy = 0;                                                             \
+      try_ip_pref = 1;                                                        \
+      goto retry_label;                                                       \
+    }                                                                         \
+  STMT_END
+
+/* Common code used in the loop within router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl.
+ *
+ * Check if the given <b>identity</b> supports extrainfo. If not, skip further
+ * checks.
+ */
+#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity)                        \
+  STMT_BEGIN                                                                  \
+    int is_trusted_extrainfo = router_digest_is_trusted_dir_type(             \
+                               (identity), EXTRAINFO_DIRINFO);                \
+    if (((type) & EXTRAINFO_DIRINFO) &&                                       \
+        !router_supports_extrainfo((identity), is_trusted_extrainfo))         \
+      continue;                                                               \
+  STMT_END
+
+/* When iterating through the routerlist, can OR address/port preference
+ * and reachability checks be skipped?
+ */
+int
+router_skip_or_reachability(const or_options_t *options, int try_ip_pref)
+{
+  /* Servers always have and prefer IPv4.
+   * And if clients are checking against the firewall for reachability only,
+   * but there's no firewall, don't bother checking */
+  return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_or());
+}
+
+/* When iterating through the routerlist, can Dir address/port preference
+ * and reachability checks be skipped?
+ */
+static int
+router_skip_dir_reachability(const or_options_t *options, int try_ip_pref)
+{
+  /* Servers always have and prefer IPv4.
+   * And if clients are checking against the firewall for reachability only,
+   * but there's no firewall, don't bother checking */
+  return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir());
+}
+
+/** Pick a random running valid directory server/mirror from our
+ * routerlist.  Arguments are as for router_pick_directory_server(), except:
+ *
+ * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of
+ * directories that we excluded for no other reason than
+ * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH.
+ */
+STATIC const routerstatus_t *
+router_pick_directory_server_impl(dirinfo_type_t type, int flags,
+                                  int *n_busy_out)
+{
+  const or_options_t *options = get_options();
+  const node_t *result;
+  smartlist_t *direct, *tunnel;
+  smartlist_t *trusted_direct, *trusted_tunnel;
+  smartlist_t *overloaded_direct, *overloaded_tunnel;
+  time_t now = time(NULL);
+  const networkstatus_t *consensus = networkstatus_get_latest_consensus();
+  const int requireother = ! (flags & PDS_ALLOW_SELF);
+  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+  const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+  int try_excluding = 1, n_excluded = 0, n_busy = 0;
+  int try_ip_pref = 1;
+
+  if (!consensus)
+    return NULL;
+
+ retry_search:
+
+  direct = smartlist_new();
+  tunnel = smartlist_new();
+  trusted_direct = smartlist_new();
+  trusted_tunnel = smartlist_new();
+  overloaded_direct = smartlist_new();
+  overloaded_tunnel = smartlist_new();
+
+  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+  const int must_have_or = directory_must_use_begindir(options);
+
+  /* Find all the running dirservers we know about. */
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
+    int is_trusted;
+    int is_overloaded;
+    const routerstatus_t *status = node->rs;
+    const country_t country = node->country;
+    if (!status)
+      continue;
+
+    if (!node->is_running || !node_is_dir(node) || !node->is_valid)
+      continue;
+    if (requireother && router_digest_is_me(node->identity))
+      continue;
+
+    SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity);
+
+    if (try_excluding &&
+        routerset_contains_routerstatus(options->ExcludeNodes, status,
+                                        country)) {
+      ++n_excluded;
+      continue;
+    }
+
+    if (router_is_already_dir_fetching_(status->addr,
+                                        &status->ipv6_addr,
+                                        status->dir_port,
+                                        no_serverdesc_fetching,
+                                        no_microdesc_fetching)) {
+      ++n_busy;
+      continue;
+    }
+
+    is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
+    is_trusted = router_digest_is_trusted_dir(node->identity);
+
+    /* Clients use IPv6 addresses if the server has one and the client
+     * prefers IPv6.
+     * Add the router if its preferred address and port are reachable.
+     * If we don't get any routers, we'll try again with the non-preferred
+     * address for each router (if any). (To ensure correct load-balancing
+     * we try routers that only have one address both times.)
+     */
+    if (!fascistfirewall || skip_or_fw ||
+        fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
+                                     try_ip_pref))
+      smartlist_add(is_trusted ? trusted_tunnel :
+                    is_overloaded ? overloaded_tunnel : tunnel, (void*)node);
+    else if (!must_have_or && (skip_dir_fw ||
+             fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION,
+                                          try_ip_pref)))
+      smartlist_add(is_trusted ? trusted_direct :
+                    is_overloaded ? overloaded_direct : direct, (void*)node);
+  } SMARTLIST_FOREACH_END(node);
+
+  if (smartlist_len(tunnel)) {
+    result = node_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR);
+  } else if (smartlist_len(overloaded_tunnel)) {
+    result = node_sl_choose_by_bandwidth(overloaded_tunnel,
+                                                 WEIGHT_FOR_DIR);
+  } else if (smartlist_len(trusted_tunnel)) {
+    /* FFFF We don't distinguish between trusteds and overloaded trusteds
+     * yet. Maybe one day we should. */
+    /* FFFF We also don't load balance over authorities yet. I think this
+     * is a feature, but it could easily be a bug. -RD */
+    result = smartlist_choose(trusted_tunnel);
+  } else if (smartlist_len(direct)) {
+    result = node_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR);
+  } else if (smartlist_len(overloaded_direct)) {
+    result = node_sl_choose_by_bandwidth(overloaded_direct,
+                                         WEIGHT_FOR_DIR);
+  } else {
+    result = smartlist_choose(trusted_direct);
+  }
+  smartlist_free(direct);
+  smartlist_free(tunnel);
+  smartlist_free(trusted_direct);
+  smartlist_free(trusted_tunnel);
+  smartlist_free(overloaded_direct);
+  smartlist_free(overloaded_tunnel);
+
+  RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+  RETRY_WITHOUT_EXCLUDE(retry_search);
+
+  if (n_busy_out)
+    *n_busy_out = n_busy;
+
+  router_picked_poor_directory_log(result ? result->rs : NULL);
+
+  return result ? result->rs : NULL;
+}
+
+/** Pick a random element from a list of dir_server_t, weighting by their
+ * <b>weight</b> field. */
+static const dir_server_t *
+dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight)
+{
+  int n = smartlist_len(servers);
+  int i;
+  double *weights_dbl;
+  uint64_t *weights_u64;
+  const dir_server_t *ds;
+
+  weights_dbl = tor_calloc(n, sizeof(double));
+  weights_u64 = tor_calloc(n, sizeof(uint64_t));
+  for (i = 0; i < n; ++i) {
+    ds = smartlist_get(servers, i);
+    weights_dbl[i] = ds->weight;
+    if (ds->is_authority)
+      weights_dbl[i] *= authority_weight;
+  }
+
+  scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL);
+  i = choose_array_element_by_weight(weights_u64, n);
+  tor_free(weights_dbl);
+  tor_free(weights_u64);
+  return (i < 0) ? NULL : smartlist_get(servers, i);
+}
+
+/** Choose randomly from among the dir_server_ts in sourcelist that
+ * are up. Flags are as for router_pick_directory_server_impl().
+ */
+static const routerstatus_t *
+router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
+                                  dirinfo_type_t type, int flags,
+                                  int *n_busy_out)
+{
+  const or_options_t *options = get_options();
+  smartlist_t *direct, *tunnel;
+  smartlist_t *overloaded_direct, *overloaded_tunnel;
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const routerstatus_t *result = NULL;
+  time_t now = time(NULL);
+  const int requireother = ! (flags & PDS_ALLOW_SELF);
+  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+  const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+  const double auth_weight = (sourcelist == fallback_dir_servers) ?
+    options->DirAuthorityFallbackRate : 1.0;
+  smartlist_t *pick_from;
+  int n_busy = 0;
+  int try_excluding = 1, n_excluded = 0;
+  int try_ip_pref = 1;
+
+  if (!sourcelist)
+    return NULL;
+
+ retry_search:
+
+  direct = smartlist_new();
+  tunnel = smartlist_new();
+  overloaded_direct = smartlist_new();
+  overloaded_tunnel = smartlist_new();
+
+  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+  const int must_have_or = directory_must_use_begindir(options);
+
+  SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
+    {
+      int is_overloaded =
+          d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now;
+      if (!d->is_running) continue;
+      if ((type & d->type) == 0)
+        continue;
+
+      SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest);
+
+      if (requireother && me && router_digest_is_me(d->digest))
+        continue;
+      if (try_excluding &&
+          routerset_contains_routerstatus(options->ExcludeNodes,
+                                          &d->fake_status, -1)) {
+        ++n_excluded;
+        continue;
+      }
+
+      if (router_is_already_dir_fetching_(d->addr,
+                                          &d->ipv6_addr,
+                                          d->dir_port,
+                                          no_serverdesc_fetching,
+                                          no_microdesc_fetching)) {
+        ++n_busy;
+        continue;
+      }
+
+      /* Clients use IPv6 addresses if the server has one and the client
+       * prefers IPv6.
+       * Add the router if its preferred address and port are reachable.
+       * If we don't get any routers, we'll try again with the non-preferred
+       * address for each router (if any). (To ensure correct load-balancing
+       * we try routers that only have one address both times.)
+       */
+      if (!fascistfirewall || skip_or_fw ||
+          fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION,
+                                             try_ip_pref))
+        smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d);
+      else if (!must_have_or && (skip_dir_fw ||
+               fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION,
+                                                  try_ip_pref)))
+        smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d);
+    }
+  SMARTLIST_FOREACH_END(d);
+
+  if (smartlist_len(tunnel)) {
+    pick_from = tunnel;
+  } else if (smartlist_len(overloaded_tunnel)) {
+    pick_from = overloaded_tunnel;
+  } else if (smartlist_len(direct)) {
+    pick_from = direct;
+  } else {
+    pick_from = overloaded_direct;
+  }
+
+  {
+    const dir_server_t *selection =
+      dirserver_choose_by_weight(pick_from, auth_weight);
+
+    if (selection)
+      result = &selection->fake_status;
+  }
+
+  smartlist_free(direct);
+  smartlist_free(tunnel);
+  smartlist_free(overloaded_direct);
+  smartlist_free(overloaded_tunnel);
+
+  RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+  RETRY_WITHOUT_EXCLUDE(retry_search);
+
+  router_picked_poor_directory_log(result);
+
+  if (n_busy_out)
+    *n_busy_out = n_busy;
+  return result;
+}
+
+/** Mark as running every dir_server_t in <b>server_list</b>. */
+static void
+mark_all_dirservers_up(smartlist_t *server_list)
+{
+  if (server_list) {
+    SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) {
+      routerstatus_t *rs;
+      node_t *node;
+      dir->is_running = 1;
+      node = node_get_mutable_by_id(dir->digest);
+      if (node)
+        node->is_running = 1;
+      rs = router_get_mutable_consensus_status_by_id(dir->digest);
+      if (rs) {
+        rs->last_dir_503_at = 0;
+        control_event_networkstatus_changed_single(rs);
+      }
+    } SMARTLIST_FOREACH_END(dir);
+  }
+  router_dir_info_changed();
+}
+
+/** Return true iff r1 and r2 have the same address and OR port. */
+int
+routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2)
+{
+  return r1->addr == r2->addr && r1->or_port == r2->or_port &&
+    tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) &&
+    r1->ipv6_orport == r2->ipv6_orport;
+}
+
+/** Reset all internal variables used to count failed downloads of network
+ * status objects. */
+void
+router_reset_status_download_failures(void)
+{
+  mark_all_dirservers_up(fallback_dir_servers);
+}
+
+/** Given a <b>router</b>, add every node_t in its family (including the
+ * node itself!) to <b>sl</b>.
+ *
+ * Note the type mismatch: This function takes a routerinfo, but adds nodes
+ * to the smartlist!
+ */
+static void
+routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
+{
+  /* XXXX MOVE ? */
+  node_t fake_node;
+  const node_t *node = node_get_by_id(router->cache_info.identity_digest);
+  if (node == NULL) {
+    memset(&fake_node, 0, sizeof(fake_node));
+    fake_node.ri = (routerinfo_t *)router;
+    memcpy(fake_node.identity, router->cache_info.identity_digest, DIGEST_LEN);
+    node = &fake_node;
+  }
+  nodelist_add_node_and_family(sl, node);
+}
+
+/** Add every suitable node from our nodelist to <b>sl</b>, so that
+ * we can pick a node for a circuit.
+ */
+void
+router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                      int need_capacity, int need_guard,
+                                      int need_desc, int pref_addr,
+                                      int direct_conn)
+{
+  const int check_reach = !router_skip_or_reachability(get_options(),
+                                                       pref_addr);
+  /* XXXX MOVE */
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
+    if (!node->is_running || !node->is_valid)
+      continue;
+    if (need_desc && !node_has_preferred_descriptor(node, direct_conn))
+      continue;
+    if (node->ri && node->ri->purpose != ROUTER_PURPOSE_GENERAL)
+      continue;
+    if (node_is_unreliable(node, need_uptime, need_capacity, need_guard))
+      continue;
+    /* Don't choose nodes if we are certain they can't do EXTEND2 cells */
+    if (node->rs && !routerstatus_version_supports_extend2_cells(node->rs, 1))
+      continue;
+    /* Don't choose nodes if we are certain they can't do ntor. */
+    if ((node->ri || node->md) && !node_has_curve25519_onion_key(node))
+      continue;
+    /* Choose a node with an OR address that matches the firewall rules */
+    if (direct_conn && check_reach &&
+        !fascist_firewall_allows_node(node,
+                                      FIREWALL_OR_CONNECTION,
+                                      pref_addr))
+      continue;
+
+    smartlist_add(sl, (void *)node);
+  } SMARTLIST_FOREACH_END(node);
+}
+
+/** Look through the routerlist until we find a router that has my key.
+ Return it. */
+const routerinfo_t *
+routerlist_find_my_routerinfo(void)
+{
+  if (!routerlist)
+    return NULL;
+
+  SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, router,
+  {
+    if (router_is_me(router))
+      return router;
+  });
+  return NULL;
+}
+
+/** Return the smaller of the router's configured BandwidthRate
+ * and its advertised capacity. */
+uint32_t
+router_get_advertised_bandwidth(const routerinfo_t *router)
+{
+  if (router->bandwidthcapacity < router->bandwidthrate)
+    return router->bandwidthcapacity;
+  return router->bandwidthrate;
+}
+
+/** Do not weight any declared bandwidth more than this much when picking
+ * routers by bandwidth. */
+#define DEFAULT_MAX_BELIEVABLE_BANDWIDTH 10000000 /* 10 MB/sec */
+
+/** Return the smaller of the router's configured BandwidthRate
+ * and its advertised capacity, capped by max-believe-bw. */
+uint32_t
+router_get_advertised_bandwidth_capped(const routerinfo_t *router)
+{
+  uint32_t result = router->bandwidthcapacity;
+  if (result > router->bandwidthrate)
+    result = router->bandwidthrate;
+  if (result > DEFAULT_MAX_BELIEVABLE_BANDWIDTH)
+    result = DEFAULT_MAX_BELIEVABLE_BANDWIDTH;
+  return result;
+}
+
+/** Given an array of double/uint64_t unions that are currently being used as
+ * doubles, convert them to uint64_t, and try to scale them linearly so as to
+ * much of the range of uint64_t. If <b>total_out</b> is provided, set it to
+ * the sum of all elements in the array _before_ scaling. */
+STATIC void
+scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in,
+                            int n_entries,
+                            uint64_t *total_out)
+{
+  double total = 0.0;
+  double scale_factor = 0.0;
+  int i;
+
+  for (i = 0; i < n_entries; ++i)
+    total += entries_in[i];
+
+  if (total > 0.0) {
+    scale_factor = ((double)INT64_MAX) / total;
+    scale_factor /= 4.0; /* make sure we're very far away from overflowing */
+  }
+
+  for (i = 0; i < n_entries; ++i)
+    entries_out[i] = tor_llround(entries_in[i] * scale_factor);
+
+  if (total_out)
+    *total_out = (uint64_t) total;
+}
+
+/** Pick a random element of <b>n_entries</b>-element array <b>entries</b>,
+ * choosing each element with a probability proportional to its (uint64_t)
+ * value, and return the index of that element.  If all elements are 0, choose
+ * an index at random. Return -1 on error.
+ */
+STATIC int
+choose_array_element_by_weight(const uint64_t *entries, int n_entries)
+{
+  int i;
+  uint64_t rand_val;
+  uint64_t total = 0;
+
+  for (i = 0; i < n_entries; ++i)
+    total += entries[i];
+
+  if (n_entries < 1)
+    return -1;
+
+  if (total == 0)
+    return crypto_rand_int(n_entries);
+
+  tor_assert(total < INT64_MAX);
+
+  rand_val = crypto_rand_uint64(total);
+
+  return select_array_member_cumulative_timei(
+                           entries, n_entries, total, rand_val);
+}
+
+/** When weighting bridges, enforce these values as lower and upper
+ * bound for believable bandwidth, because there is no way for us
+ * to verify a bridge's bandwidth currently. */
+#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000  /* 20 kB/sec */
+#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */
+
+/** Return the smaller of the router's configured BandwidthRate
+ * and its advertised capacity, making sure to stay within the
+ * interval between bridge-min-believe-bw and
+ * bridge-max-believe-bw. */
+static uint32_t
+bridge_get_advertised_bandwidth_bounded(routerinfo_t *router)
+{
+  uint32_t result = router->bandwidthcapacity;
+  if (result > router->bandwidthrate)
+    result = router->bandwidthrate;
+  if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH)
+    result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH;
+  else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH)
+    result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH;
+  return result;
+}
+
+/** Return bw*1000, unless bw*1000 would overflow, in which case return
+ * INT32_MAX. */
+static inline int32_t
+kb_to_bytes(uint32_t bw)
+{
+  return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000;
+}
+
+/** Helper function:
+ * choose a random element of smartlist <b>sl</b> of nodes, weighted by
+ * the advertised bandwidth of each element using the consensus
+ * bandwidth weights.
+ *
+ * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all
+ * nodes' bandwidth equally regardless of their Exit status, since there may
+ * be some in the list because they exit to obscure ports. If
+ * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight
+ * exit-node's bandwidth less depending on the smallness of the fraction of
+ * Exit-to-total bandwidth.  If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a
+ * guard node: consider all guard's bandwidth equally. Otherwise, weight
+ * guards proportionally less.
+ */
+static const node_t *
+smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl,
+                                           bandwidth_weight_rule_t rule)
+{
+  double *bandwidths_dbl=NULL;
+  uint64_t *bandwidths_u64=NULL;
+
+  if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl, NULL) < 0)
+    return NULL;
+
+  bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t));
+  scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl,
+                              smartlist_len(sl), NULL);
+
+  {
+    int idx = choose_array_element_by_weight(bandwidths_u64,
+                                             smartlist_len(sl));
+    tor_free(bandwidths_dbl);
+    tor_free(bandwidths_u64);
+    return idx < 0 ? NULL : smartlist_get(sl, idx);
+  }
+}
+
+/** Given a list of routers and a weighting rule as in
+ * smartlist_choose_node_by_bandwidth_weights, compute weighted bandwidth
+ * values for each node and store them in a freshly allocated
+ * *<b>bandwidths_out</b> of the same length as <b>sl</b>, and holding results
+ * as doubles. If <b>total_bandwidth_out</b> is non-NULL, set it to the total
+ * of all the bandwidths.
+ * Return 0 on success, -1 on failure. */
+static int
+compute_weighted_bandwidths(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule,
+                            double **bandwidths_out,
+                            double *total_bandwidth_out)
+{
+  int64_t weight_scale;
+  double Wg = -1, Wm = -1, We = -1, Wd = -1;
+  double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1;
+  guardfraction_bandwidth_t guardfraction_bw;
+  double *bandwidths = NULL;
+  double total_bandwidth = 0.0;
+
+  tor_assert(sl);
+  tor_assert(bandwidths_out);
+
+  /* Can't choose exit and guard at same time */
+  tor_assert(rule == NO_WEIGHTING ||
+             rule == WEIGHT_FOR_EXIT ||
+             rule == WEIGHT_FOR_GUARD ||
+             rule == WEIGHT_FOR_MID ||
+             rule == WEIGHT_FOR_DIR);
+
+  *bandwidths_out = NULL;
+
+  if (total_bandwidth_out) {
+    *total_bandwidth_out = 0.0;
+  }
+
+  if (smartlist_len(sl) == 0) {
+    log_info(LD_CIRC,
+             "Empty routerlist passed in to consensus weight node "
+             "selection for rule %s",
+             bandwidth_weight_rule_to_string(rule));
+    return -1;
+  }
+
+  weight_scale = networkstatus_get_weight_scale_param(NULL);
+
+  if (rule == WEIGHT_FOR_GUARD) {
+    Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */
+    We = 0;
+    Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1);
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_MID) {
+    Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1);
+    We = networkstatus_get_bw_weight(NULL, "Wme", -1);
+    Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1);
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_EXIT) {
+    // Guards CAN be exits if they have weird exit policies
+    // They are d then I guess...
+    We = networkstatus_get_bw_weight(NULL, "Wee", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */
+    Wd = networkstatus_get_bw_weight(NULL, "Wed", -1);
+    Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_DIR) {
+    We = networkstatus_get_bw_weight(NULL, "Wbe", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1);
+    Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1);
+    Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1);
+
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  } else if (rule == NO_WEIGHTING) {
+    Wg = Wm = We = Wd = weight_scale;
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  }
+
+  if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0
+      || Web < 0) {
+    log_debug(LD_CIRC,
+              "Got negative bandwidth weights. Defaulting to naive selection"
+              " algorithm.");
+    Wg = Wm = We = Wd = weight_scale;
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  }
+
+  Wg /= weight_scale;
+  Wm /= weight_scale;
+  We /= weight_scale;
+  Wd /= weight_scale;
+
+  Wgb /= weight_scale;
+  Wmb /= weight_scale;
+  Web /= weight_scale;
+  Wdb /= weight_scale;
+
+  bandwidths = tor_calloc(smartlist_len(sl), sizeof(double));
+
+  // Cycle through smartlist and total the bandwidth.
+  static int warned_missing_bw = 0;
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0;
+    double weight = 1;
+    double weight_without_guard_flag = 0; /* Used for guardfraction */
+    double final_weight = 0;
+    is_exit = node->is_exit && ! node->is_bad_exit;
+    is_guard = node->is_possible_guard;
+    is_dir = node_is_dir(node);
+    if (node->rs) {
+      if (!node->rs->has_bandwidth) {
+        /* This should never happen, unless all the authorities downgrade
+         * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */
+        if (! warned_missing_bw) {
+          log_warn(LD_BUG,
+                 "Consensus is missing some bandwidths. Using a naive "
+                 "router selection algorithm");
+          warned_missing_bw = 1;
+        }
+        this_bw = 30000; /* Chosen arbitrarily */
+      } else {
+        this_bw = kb_to_bytes(node->rs->bandwidth_kb);
+      }
+    } else if (node->ri) {
+      /* bridge or other descriptor not in our consensus */
+      this_bw = bridge_get_advertised_bandwidth_bounded(node->ri);
+    } else {
+      /* We can't use this one. */
+      continue;
+    }
+
+    if (is_guard && is_exit) {
+      weight = (is_dir ? Wdb*Wd : Wd);
+      weight_without_guard_flag = (is_dir ? Web*We : We);
+    } else if (is_guard) {
+      weight = (is_dir ? Wgb*Wg : Wg);
+      weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm);
+    } else if (is_exit) {
+      weight = (is_dir ? Web*We : We);
+    } else { // middle
+      weight = (is_dir ? Wmb*Wm : Wm);
+    }
+    /* These should be impossible; but overflows here would be bad, so let's
+     * make sure. */
+    if (this_bw < 0)
+      this_bw = 0;
+    if (weight < 0.0)
+      weight = 0.0;
+    if (weight_without_guard_flag < 0.0)
+      weight_without_guard_flag = 0.0;
+
+    /* If guardfraction information is available in the consensus, we
+     * want to calculate this router's bandwidth according to its
+     * guardfraction. Quoting from proposal236:
+     *
+     *    Let Wpf denote the weight from the 'bandwidth-weights' line a
+     *    client would apply to N for position p if it had the guard
+     *    flag, Wpn the weight if it did not have the guard flag, and B the
+     *    measured bandwidth of N in the consensus.  Then instead of choosing
+     *    N for position p proportionally to Wpf*B or Wpn*B, clients should
+     *    choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
+     */
+    if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) {
+      /* XXX The assert should actually check for is_guard. However,
+       * that crashes dirauths because of #13297. This should be
+       * equivalent: */
+      tor_assert(node->rs->is_possible_guard);
+
+      guard_get_guardfraction_bandwidth(&guardfraction_bw,
+                                        this_bw,
+                                        node->rs->guardfraction_percentage);
+
+      /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */
+      final_weight =
+        guardfraction_bw.guard_bw * weight +
+        guardfraction_bw.non_guard_bw * weight_without_guard_flag;
+
+      log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)",
+                node->rs->nickname, final_weight, weight*this_bw,
+                bandwidth_weight_rule_to_string(rule));
+    } else { /* no guardfraction information. calculate the weight normally. */
+      final_weight = weight*this_bw;
+    }
+
+    bandwidths[node_sl_idx] = final_weight;
+    total_bandwidth += final_weight;
+  } SMARTLIST_FOREACH_END(node);
+
+  log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based "
+            "on weights "
+            "Wg=%f Wm=%f We=%f Wd=%f with total bw %f",
+            bandwidth_weight_rule_to_string(rule),
+            Wg, Wm, We, Wd, total_bandwidth);
+
+  *bandwidths_out = bandwidths;
+
+  if (total_bandwidth_out) {
+    *total_bandwidth_out = total_bandwidth;
+  }
+
+  return 0;
+}
+
+/** For all nodes in <b>sl</b>, return the fraction of those nodes, weighted
+ * by their weighted bandwidths with rule <b>rule</b>, for which we have
+ * descriptors.
+ *
+ * If <b>for_direct_connect</b> is true, we intend to connect to the node
+ * directly, as the first hop of a circuit; otherwise, we intend to connect
+ * to it indirectly, or use it as if we were connecting to it indirectly. */
+double
+frac_nodes_with_descriptors(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule,
+                            int for_direct_conn)
+{
+  double *bandwidths = NULL;
+  double total, present;
+
+  if (smartlist_len(sl) == 0)
+    return 0.0;
+
+  if (compute_weighted_bandwidths(sl, rule, &bandwidths, &total) < 0 ||
+      total <= 0.0) {
+    int n_with_descs = 0;
+    SMARTLIST_FOREACH(sl, const node_t *, node, {
+      if (node_has_preferred_descriptor(node, for_direct_conn))
+        n_with_descs++;
+    });
+    tor_free(bandwidths);
+    return ((double)n_with_descs) / smartlist_len(sl);
+  }
+
+  present = 0.0;
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    if (node_has_preferred_descriptor(node, for_direct_conn))
+      present += bandwidths[node_sl_idx];
+  } SMARTLIST_FOREACH_END(node);
+
+  tor_free(bandwidths);
+
+  return present / total;
+}
+
+/** Choose a random element of status list <b>sl</b>, weighted by
+ * the advertised bandwidth of each node */
+const node_t *
+node_sl_choose_by_bandwidth(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule)
+{ /*XXXX MOVE */
+  return smartlist_choose_node_by_bandwidth_weights(sl, rule);
+}
+
+/** Return a random running node from the nodelist. Never
+ * pick a node that is in
+ * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
+ * even if they are the only nodes available.
+ * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than
+ * a minimum uptime, return one of those.
+ * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the
+ * advertised capacity of each router.
+ * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers.
+ * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if
+ * picking an exit node, otherwise we weight bandwidths for picking a relay
+ * node (that is, possibly discounting exit nodes).
+ * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that
+ * have a routerinfo or microdescriptor -- that is, enough info to be
+ * used to build a circuit.
+ * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that
+ * have an address that is preferred by the ClientPreferIPv6ORPort setting
+ * (regardless of this flag, we exclude nodes that aren't allowed by the
+ * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0).
+ */
+const node_t *
+router_choose_random_node(smartlist_t *excludedsmartlist,
+                          routerset_t *excludedset,
+                          router_crn_flags_t flags)
+{ /* XXXX MOVE */
+  const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
+  const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
+  const int need_guard = (flags & CRN_NEED_GUARD) != 0;
+  const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0;
+  const int need_desc = (flags & CRN_NEED_DESC) != 0;
+  const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
+  const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
+  const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
+
+  smartlist_t *sl=smartlist_new(),
+    *excludednodes=smartlist_new();
+  const node_t *choice = NULL;
+  const routerinfo_t *r;
+  bandwidth_weight_rule_t rule;
+
+  tor_assert(!(weight_for_exit && need_guard));
+  rule = weight_for_exit ? WEIGHT_FOR_EXIT :
+    (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
+
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+    if (node_allows_single_hop_exits(node)) {
+      /* Exclude relays that allow single hop exit circuits. This is an
+       * obsolete option since 0.2.9.2-alpha and done by default in
+       * 0.3.1.0-alpha. */
+      smartlist_add(excludednodes, node);
+    } else if (rendezvous_v3 &&
+               !node_supports_v3_rendezvous_point(node)) {
+      /* Exclude relays that do not support to rendezvous for a hidden service
+       * version 3. */
+      smartlist_add(excludednodes, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* If the node_t is not found we won't be to exclude ourself but we
+   * won't be able to pick ourself in router_choose_random_node() so
+   * this is fine to at least try with our routerinfo_t object. */
+  if ((r = router_get_my_routerinfo()))
+    routerlist_add_node_and_family(excludednodes, r);
+
+  router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity,
+                                        need_guard, need_desc, pref_addr,
+                                        direct_conn);
+  log_debug(LD_CIRC,
+           "We found %d running nodes.",
+            smartlist_len(sl));
+
+  smartlist_subtract(sl,excludednodes);
+  log_debug(LD_CIRC,
+            "We removed %d excludednodes, leaving %d nodes.",
+            smartlist_len(excludednodes),
+            smartlist_len(sl));
+
+  if (excludedsmartlist) {
+    smartlist_subtract(sl,excludedsmartlist);
+    log_debug(LD_CIRC,
+              "We removed %d excludedsmartlist, leaving %d nodes.",
+              smartlist_len(excludedsmartlist),
+              smartlist_len(sl));
+  }
+  if (excludedset) {
+    routerset_subtract_nodes(sl,excludedset);
+    log_debug(LD_CIRC,
+              "We removed excludedset, leaving %d nodes.",
+              smartlist_len(sl));
+  }
+
+  // Always weight by bandwidth
+  choice = node_sl_choose_by_bandwidth(sl, rule);
+
+  smartlist_free(sl);
+  if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) {
+    /* try once more -- recurse but with fewer restrictions. */
+    log_info(LD_CIRC,
+             "We couldn't find any live%s%s%s routers; falling back "
+             "to list of all routers.",
+             need_capacity?", fast":"",
+             need_uptime?", stable":"",
+             need_guard?", guard":"");
+    flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD|
+                CRN_PREF_ADDR);
+    choice = router_choose_random_node(
+                     excludedsmartlist, excludedset, flags);
+  }
+  smartlist_free(excludednodes);
+  if (!choice) {
+    log_warn(LD_CIRC,
+             "No available nodes when trying to choose node. Failing.");
+  }
+  return choice;
+}
+
+/** Helper: given an extended nickname in <b>hexdigest</b> try to decode it.
+ * Return 0 on success, -1 on failure.  Store the result into the
+ * DIGEST_LEN-byte buffer at <b>digest_out</b>, the single character at
+ * <b>nickname_qualifier_char_out</b>, and the MAXNICKNAME_LEN+1-byte buffer
+ * at <b>nickname_out</b>.
+ *
+ * The recognized format is:
+ *   HexName = Dollar? HexDigest NamePart?
+ *   Dollar = '?'
+ *   HexDigest = HexChar*20
+ *   HexChar = 'a'..'f' | 'A'..'F' | '0'..'9'
+ *   NamePart = QualChar Name
+ *   QualChar = '=' | '~'
+ *   Name = NameChar*(1..MAX_NICKNAME_LEN)
+ *   NameChar = Any ASCII alphanumeric character
+ */
+int
+hex_digest_nickname_decode(const char *hexdigest,
+                           char *digest_out,
+                           char *nickname_qualifier_char_out,
+                           char *nickname_out)
+{
+  size_t len;
+
+  tor_assert(hexdigest);
+  if (hexdigest[0] == '$')
+    ++hexdigest;
+
+  len = strlen(hexdigest);
+  if (len < HEX_DIGEST_LEN) {
+    return -1;
+  } else if (len > HEX_DIGEST_LEN && (hexdigest[HEX_DIGEST_LEN] == '=' ||
+                                    hexdigest[HEX_DIGEST_LEN] == '~') &&
+           len <= HEX_DIGEST_LEN+1+MAX_NICKNAME_LEN) {
+    *nickname_qualifier_char_out = hexdigest[HEX_DIGEST_LEN];
+    strlcpy(nickname_out, hexdigest+HEX_DIGEST_LEN+1 , MAX_NICKNAME_LEN+1);
+  } else if (len == HEX_DIGEST_LEN) {
+    ;
+  } else {
+    return -1;
+  }
+
+  if (base16_decode(digest_out, DIGEST_LEN,
+                    hexdigest, HEX_DIGEST_LEN) != DIGEST_LEN)
+    return -1;
+  return 0;
+}
+
+/** Helper: Return true iff the <b>identity_digest</b> and <b>nickname</b>
+ * combination of a router, encoded in hexadecimal, matches <b>hexdigest</b>
+ * (which is optionally prefixed with a single dollar sign).  Return false if
+ * <b>hexdigest</b> is malformed, or it doesn't match.  */
+int
+hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest,
+                            const char *nickname)
+{
+  char digest[DIGEST_LEN];
+  char nn_char='\0';
+  char nn_buf[MAX_NICKNAME_LEN+1];
+
+  if (hex_digest_nickname_decode(hexdigest, digest, &nn_char, nn_buf) == -1)
+    return 0;
+
+  if (nn_char == '=') {
+    return 0;
+  }
+
+  if (nn_char == '~') {
+    if (!nickname)     // XXX This seems wrong. -NM
+      return 0;
+    if (strcasecmp(nn_buf, nickname))
+      return 0;
+  }
+
+  return tor_memeq(digest, identity_digest, DIGEST_LEN);
+}
+
+/** Return true iff <b>digest</b> is the digest of the identity key of a
+ * trusted directory matching at least one bit of <b>type</b>.  If <b>type</b>
+ * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */
+int
+router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
+{
+  if (!trusted_dir_servers)
+    return 0;
+  if (authdir_mode(get_options()) && router_digest_is_me(digest))
+    return 1;
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
+    if (tor_memeq(digest, ent->digest, DIGEST_LEN)) {
+      return (!type) || ((type & ent->type) != 0);
+    });
+  return 0;
+}
+
+/** If hexdigest is correctly formed, base16_decode it into
+ * digest, which must have DIGEST_LEN space in it.
+ * Return 0 on success, -1 on failure.
+ */
+int
+hexdigest_to_digest(const char *hexdigest, char *digest)
+{
+  if (hexdigest[0]=='$')
+    ++hexdigest;
+  if (strlen(hexdigest) < HEX_DIGEST_LEN ||
+      base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN) != DIGEST_LEN)
+    return -1;
+  return 0;
+}
+
+/** As router_get_by_id_digest,but return a pointer that you're allowed to
+ * modify */
+routerinfo_t *
+router_get_mutable_by_digest(const char *digest)
+{
+  tor_assert(digest);
+
+  if (!routerlist) return NULL;
+
+  // routerlist_assert_ok(routerlist);
+
+  return rimap_get(routerlist->identity_map, digest);
+}
+
+/** Return the router in our routerlist whose 20-byte key digest
+ * is <b>digest</b>.  Return NULL if no such router is known. */
+const routerinfo_t *
+router_get_by_id_digest(const char *digest)
+{
+  return router_get_mutable_by_digest(digest);
+}
+
+/** Return the router in our routerlist whose 20-byte descriptor
+ * is <b>digest</b>.  Return NULL if no such router is known. */
+signed_descriptor_t *
+router_get_by_descriptor_digest(const char *digest)
+{
+  tor_assert(digest);
+
+  if (!routerlist) return NULL;
+
+  return sdmap_get(routerlist->desc_digest_map, digest);
+}
+
+/** Return the signed descriptor for the router in our routerlist whose
+ * 20-byte extra-info digest is <b>digest</b>.  Return NULL if no such router
+ * is known. */
+MOCK_IMPL(signed_descriptor_t *,
+router_get_by_extrainfo_digest,(const char *digest))
+{
+  tor_assert(digest);
+
+  if (!routerlist) return NULL;
+
+  return sdmap_get(routerlist->desc_by_eid_map, digest);
+}
+
+/** Return the signed descriptor for the extrainfo_t in our routerlist whose
+ * extra-info-digest is <b>digest</b>. Return NULL if no such extra-info
+ * document is known. */
+MOCK_IMPL(signed_descriptor_t *,
+extrainfo_get_by_descriptor_digest,(const char *digest))
+{
+  extrainfo_t *ei;
+  tor_assert(digest);
+  if (!routerlist) return NULL;
+  ei = eimap_get(routerlist->extra_info_map, digest);
+  return ei ? &ei->cache_info : NULL;
+}
+
+/** Return a pointer to the signed textual representation of a descriptor.
+ * The returned string is not guaranteed to be NUL-terminated: the string's
+ * length will be in desc-\>signed_descriptor_len.
+ *
+ * If <b>with_annotations</b> is set, the returned string will include
+ * the annotations
+ * (if any) preceding the descriptor.  This will increase the length of the
+ * string by desc-\>annotations_len.
+ *
+ * The caller must not free the string returned.
+ */
+static const char *
+signed_descriptor_get_body_impl(const signed_descriptor_t *desc,
+                                int with_annotations)
+{
+  const char *r = NULL;
+  size_t len = desc->signed_descriptor_len;
+  off_t offset = desc->saved_offset;
+  if (with_annotations)
+    len += desc->annotations_len;
+  else
+    offset += desc->annotations_len;
+
+  tor_assert(len > 32);
+  if (desc->saved_location == SAVED_IN_CACHE && routerlist) {
+    desc_store_t *store = desc_get_store(router_get_routerlist(), desc);
+    if (store && store->mmap) {
+      tor_assert(desc->saved_offset + len <= store->mmap->size);
+      r = store->mmap->data + offset;
+    } else if (store) {
+      log_err(LD_DIR, "We couldn't read a descriptor that is supposedly "
+              "mmaped in our cache.  Is another process running in our data "
+              "directory?  Exiting.");
+      exit(1); // XXXX bad exit: should recover.
+    }
+  }
+  if (!r) /* no mmap, or not in cache. */
+    r = desc->signed_descriptor_body +
+      (with_annotations ? 0 : desc->annotations_len);
+
+  tor_assert(r);
+  if (!with_annotations) {
+    if (fast_memcmp("router ", r, 7) && fast_memcmp("extra-info ", r, 11)) {
+      char *cp = tor_strndup(r, 64);
+      log_err(LD_DIR, "descriptor at %p begins with unexpected string %s.  "
+              "Is another process running in our data directory?  Exiting.",
+              desc, escaped(cp));
+      exit(1); // XXXX bad exit: should recover.
+    }
+  }
+
+  return r;
+}
+
+/** Return a pointer to the signed textual representation of a descriptor.
+ * The returned string is not guaranteed to be NUL-terminated: the string's
+ * length will be in desc-\>signed_descriptor_len.
+ *
+ * The caller must not free the string returned.
+ */
+const char *
+signed_descriptor_get_body(const signed_descriptor_t *desc)
+{
+  return signed_descriptor_get_body_impl(desc, 0);
+}
+
+/** As signed_descriptor_get_body(), but points to the beginning of the
+ * annotations section rather than the beginning of the descriptor. */
+const char *
+signed_descriptor_get_annotations(const signed_descriptor_t *desc)
+{
+  return signed_descriptor_get_body_impl(desc, 1);
+}
+
+/** Return the current list of all known routers. */
+routerlist_t *
+router_get_routerlist(void)
+{
+  if (PREDICT_UNLIKELY(!routerlist)) {
+    routerlist = tor_malloc_zero(sizeof(routerlist_t));
+    routerlist->routers = smartlist_new();
+    routerlist->old_routers = smartlist_new();
+    routerlist->identity_map = rimap_new();
+    routerlist->desc_digest_map = sdmap_new();
+    routerlist->desc_by_eid_map = sdmap_new();
+    routerlist->extra_info_map = eimap_new();
+
+    routerlist->desc_store.fname_base = "cached-descriptors";
+    routerlist->extrainfo_store.fname_base = "cached-extrainfo";
+
+    routerlist->desc_store.type = ROUTER_STORE;
+    routerlist->extrainfo_store.type = EXTRAINFO_STORE;
+
+    routerlist->desc_store.description = "router descriptors";
+    routerlist->extrainfo_store.description = "extra-info documents";
+  }
+  return routerlist;
+}
+
+/** Free all storage held by <b>router</b>. */
+void
+routerinfo_free_(routerinfo_t *router)
+{
+  if (!router)
+    return;
+
+  tor_free(router->cache_info.signed_descriptor_body);
+  tor_free(router->nickname);
+  tor_free(router->platform);
+  tor_free(router->protocol_list);
+  tor_free(router->contact_info);
+  if (router->onion_pkey)
+    crypto_pk_free(router->onion_pkey);
+  tor_free(router->onion_curve25519_pkey);
+  if (router->identity_pkey)
+    crypto_pk_free(router->identity_pkey);
+  tor_cert_free(router->cache_info.signing_key_cert);
+  if (router->declared_family) {
+    SMARTLIST_FOREACH(router->declared_family, char *, s, tor_free(s));
+    smartlist_free(router->declared_family);
+  }
+  addr_policy_list_free(router->exit_policy);
+  short_policy_free(router->ipv6_exit_policy);
+
+  memset(router, 77, sizeof(routerinfo_t));
+
+  tor_free(router);
+}
+
+/** Release all storage held by <b>extrainfo</b> */
+void
+extrainfo_free_(extrainfo_t *extrainfo)
+{
+  if (!extrainfo)
+    return;
+  tor_cert_free(extrainfo->cache_info.signing_key_cert);
+  tor_free(extrainfo->cache_info.signed_descriptor_body);
+  tor_free(extrainfo->pending_sig);
+
+  memset(extrainfo, 88, sizeof(extrainfo_t)); /* debug bad memory usage */
+  tor_free(extrainfo);
+}
+
+#define signed_descriptor_free(val) \
+  FREE_AND_NULL(signed_descriptor_t, signed_descriptor_free_, (val))
+
+/** Release storage held by <b>sd</b>. */
+static void
+signed_descriptor_free_(signed_descriptor_t *sd)
+{
+  if (!sd)
+    return;
+
+  tor_free(sd->signed_descriptor_body);
+  tor_cert_free(sd->signing_key_cert);
+
+  memset(sd, 99, sizeof(signed_descriptor_t)); /* Debug bad mem usage */
+  tor_free(sd);
+}
+
+/** Reset the given signed descriptor <b>sd</b> by freeing the allocated
+ * memory inside the object and by zeroing its content. */
+static void
+signed_descriptor_reset(signed_descriptor_t *sd)
+{
+  tor_assert(sd);
+  tor_free(sd->signed_descriptor_body);
+  tor_cert_free(sd->signing_key_cert);
+  memset(sd, 0, sizeof(*sd));
+}
+
+/** Copy src into dest, and steal all references inside src so that when
+ * we free src, we don't mess up dest. */
+static void
+signed_descriptor_move(signed_descriptor_t *dest,
+                       signed_descriptor_t *src)
+{
+  tor_assert(dest != src);
+  /* Cleanup destination object before overwriting it.*/
+  signed_descriptor_reset(dest);
+  memcpy(dest, src, sizeof(signed_descriptor_t));
+  src->signed_descriptor_body = NULL;
+  src->signing_key_cert = NULL;
+  dest->routerlist_index = -1;
+}
+
+/** Extract a signed_descriptor_t from a general routerinfo, and free the
+ * routerinfo.
+ */
+static signed_descriptor_t *
+signed_descriptor_from_routerinfo(routerinfo_t *ri)
+{
+  signed_descriptor_t *sd;
+  tor_assert(ri->purpose == ROUTER_PURPOSE_GENERAL);
+  sd = tor_malloc_zero(sizeof(signed_descriptor_t));
+  signed_descriptor_move(sd, &ri->cache_info);
+  routerinfo_free(ri);
+  return sd;
+}
+
+/** Helper: free the storage held by the extrainfo_t in <b>e</b>. */
+static void
+extrainfo_free_void(void *e)
+{
+  extrainfo_free_(e);
+}
+
+/** Free all storage held by a routerlist <b>rl</b>. */
+void
+routerlist_free_(routerlist_t *rl)
+{
+  if (!rl)
+    return;
+  rimap_free(rl->identity_map, NULL);
+  sdmap_free(rl->desc_digest_map, NULL);
+  sdmap_free(rl->desc_by_eid_map, NULL);
+  eimap_free(rl->extra_info_map, extrainfo_free_void);
+  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
+                    routerinfo_free(r));
+  SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd,
+                    signed_descriptor_free(sd));
+  smartlist_free(rl->routers);
+  smartlist_free(rl->old_routers);
+  if (rl->desc_store.mmap) {
+    int res = tor_munmap_file(routerlist->desc_store.mmap);
+    if (res != 0) {
+      log_warn(LD_FS, "Failed to munmap routerlist->desc_store.mmap");
+    }
+  }
+  if (rl->extrainfo_store.mmap) {
+    int res = tor_munmap_file(routerlist->extrainfo_store.mmap);
+    if (res != 0) {
+      log_warn(LD_FS, "Failed to munmap routerlist->extrainfo_store.mmap");
+    }
+  }
+  tor_free(rl);
+
+  router_dir_info_changed();
+}
+
+/** Log information about how much memory is being used for routerlist,
+ * at log level <b>severity</b>. */
+void
+dump_routerlist_mem_usage(int severity)
+{
+  uint64_t livedescs = 0;
+  uint64_t olddescs = 0;
+  if (!routerlist)
+    return;
+  SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, r,
+                    livedescs += r->cache_info.signed_descriptor_len);
+  SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd,
+                    olddescs += sd->signed_descriptor_len);
+
+  tor_log(severity, LD_DIR,
+      "In %d live descriptors: %"PRIu64" bytes.  "
+      "In %d old descriptors: %"PRIu64" bytes.",
+      smartlist_len(routerlist->routers), (livedescs),
+      smartlist_len(routerlist->old_routers), (olddescs));
+}
+
+/** Debugging helper: If <b>idx</b> is nonnegative, assert that <b>ri</b> is
+ * in <b>sl</b> at position <b>idx</b>. Otherwise, search <b>sl</b> for
+ * <b>ri</b>.  Return the index of <b>ri</b> in <b>sl</b>, or -1 if <b>ri</b>
+ * is not in <b>sl</b>. */
+static inline int
+routerlist_find_elt_(smartlist_t *sl, void *ri, int idx)
+{
+  if (idx < 0) {
+    idx = -1;
+    SMARTLIST_FOREACH(sl, routerinfo_t *, r,
+                      if (r == ri) {
+                        idx = r_sl_idx;
+                        break;
+                      });
+  } else {
+    tor_assert(idx < smartlist_len(sl));
+    tor_assert(smartlist_get(sl, idx) == ri);
+  };
+  return idx;
+}
+
+/** Insert an item <b>ri</b> into the routerlist <b>rl</b>, updating indices
+ * as needed.  There must be no previous member of <b>rl</b> with the same
+ * identity digest as <b>ri</b>: If there is, call routerlist_replace
+ * instead.
+ */
+static void
+routerlist_insert(routerlist_t *rl, routerinfo_t *ri)
+{
+  routerinfo_t *ri_old;
+  signed_descriptor_t *sd_old;
+  {
+    const routerinfo_t *ri_generated = router_get_my_routerinfo();
+    tor_assert(ri_generated != ri);
+  }
+  tor_assert(ri->cache_info.routerlist_index == -1);
+
+  ri_old = rimap_set(rl->identity_map, ri->cache_info.identity_digest, ri);
+  tor_assert(!ri_old);
+
+  sd_old = sdmap_set(rl->desc_digest_map,
+                     ri->cache_info.signed_descriptor_digest,
+                     &(ri->cache_info));
+  if (sd_old) {
+    int idx = sd_old->routerlist_index;
+    sd_old->routerlist_index = -1;
+    smartlist_del(rl->old_routers, idx);
+    if (idx < smartlist_len(rl->old_routers)) {
+       signed_descriptor_t *d = smartlist_get(rl->old_routers, idx);
+       d->routerlist_index = idx;
+    }
+    rl->desc_store.bytes_dropped += sd_old->signed_descriptor_len;
+    sdmap_remove(rl->desc_by_eid_map, sd_old->extra_info_digest);
+    signed_descriptor_free(sd_old);
+  }
+
+  if (!tor_digest_is_zero(ri->cache_info.extra_info_digest))
+    sdmap_set(rl->desc_by_eid_map, ri->cache_info.extra_info_digest,
+              &ri->cache_info);
+  smartlist_add(rl->routers, ri);
+  ri->cache_info.routerlist_index = smartlist_len(rl->routers) - 1;
+  nodelist_set_routerinfo(ri, NULL);
+  router_dir_info_changed();
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Adds the extrainfo_t <b>ei</b> to the routerlist <b>rl</b>, if there is a
+ * corresponding router in rl-\>routers or rl-\>old_routers.  Return the status
+ * of inserting <b>ei</b>.  Free <b>ei</b> if it isn't inserted. */
+MOCK_IMPL(STATIC was_router_added_t,
+extrainfo_insert,(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible))
+{
+  was_router_added_t r;
+  const char *compatibility_error_msg;
+  routerinfo_t *ri = rimap_get(rl->identity_map,
+                               ei->cache_info.identity_digest);
+  signed_descriptor_t *sd =
+    sdmap_get(rl->desc_by_eid_map, ei->cache_info.signed_descriptor_digest);
+  extrainfo_t *ei_tmp;
+  const int severity = warn_if_incompatible ? LOG_WARN : LOG_INFO;
+
+  {
+    extrainfo_t *ei_generated = router_get_my_extrainfo();
+    tor_assert(ei_generated != ei);
+  }
+
+  if (!ri) {
+    /* This router is unknown; we can't even verify the signature. Give up.*/
+    r = ROUTER_NOT_IN_CONSENSUS;
+    goto done;
+  }
+  if (! sd) {
+    /* The extrainfo router doesn't have a known routerdesc to attach it to.
+     * This just won't work. */;
+    static ratelim_t no_sd_ratelim = RATELIM_INIT(1800);
+    r = ROUTER_BAD_EI;
+    log_fn_ratelim(&no_sd_ratelim, severity, LD_BUG,
+                   "No entry found in extrainfo map.");
+    goto done;
+  }
+  if (tor_memneq(ei->cache_info.signed_descriptor_digest,
+                 sd->extra_info_digest, DIGEST_LEN)) {
+    static ratelim_t digest_mismatch_ratelim = RATELIM_INIT(1800);
+    /* The sd we got from the map doesn't match the digest we used to look
+     * it up. This makes no sense. */
+    r = ROUTER_BAD_EI;
+    log_fn_ratelim(&digest_mismatch_ratelim, severity, LD_BUG,
+                     "Mismatch in digest in extrainfo map.");
+    goto done;
+  }
+  if (routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, sd,
+                                             &compatibility_error_msg)) {
+    char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1];
+    r = (ri->cache_info.extrainfo_is_bogus) ?
+      ROUTER_BAD_EI : ROUTER_NOT_IN_CONSENSUS;
+
+    base16_encode(d1, sizeof(d1), ri->cache_info.identity_digest, DIGEST_LEN);
+    base16_encode(d2, sizeof(d2), ei->cache_info.identity_digest, DIGEST_LEN);
+
+    log_fn(severity,LD_DIR,
+           "router info incompatible with extra info (ri id: %s, ei id %s, "
+           "reason: %s)", d1, d2, compatibility_error_msg);
+
+    goto done;
+  }
+
+  /* Okay, if we make it here, we definitely have a router corresponding to
+   * this extrainfo. */
+
+  ei_tmp = eimap_set(rl->extra_info_map,
+                     ei->cache_info.signed_descriptor_digest,
+                     ei);
+  r = ROUTER_ADDED_SUCCESSFULLY;
+  if (ei_tmp) {
+    rl->extrainfo_store.bytes_dropped +=
+      ei_tmp->cache_info.signed_descriptor_len;
+    extrainfo_free(ei_tmp);
+  }
+
+ done:
+  if (r != ROUTER_ADDED_SUCCESSFULLY)
+    extrainfo_free(ei);
+
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+  return r;
+}
+
+#define should_cache_old_descriptors() \
+  directory_caches_dir_info(get_options())
+
+/** If we're a directory cache and routerlist <b>rl</b> doesn't have
+ * a copy of router <b>ri</b> yet, add it to the list of old (not
+ * recommended but still served) descriptors. Else free it. */
+static void
+routerlist_insert_old(routerlist_t *rl, routerinfo_t *ri)
+{
+  {
+    const routerinfo_t *ri_generated = router_get_my_routerinfo();
+    tor_assert(ri_generated != ri);
+  }
+  tor_assert(ri->cache_info.routerlist_index == -1);
+
+  if (should_cache_old_descriptors() &&
+      ri->purpose == ROUTER_PURPOSE_GENERAL &&
+      !sdmap_get(rl->desc_digest_map,
+                 ri->cache_info.signed_descriptor_digest)) {
+    signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri);
+    sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd);
+    smartlist_add(rl->old_routers, sd);
+    sd->routerlist_index = smartlist_len(rl->old_routers)-1;
+    if (!tor_digest_is_zero(sd->extra_info_digest))
+      sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd);
+  } else {
+    routerinfo_free(ri);
+  }
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Remove an item <b>ri</b> from the routerlist <b>rl</b>, updating indices
+ * as needed. If <b>idx</b> is nonnegative and smartlist_get(rl-&gt;routers,
+ * idx) == ri, we don't need to do a linear search over the list to decide
+ * which to remove.  We fill the gap in rl-&gt;routers with a later element in
+ * the list, if any exists. <b>ri</b> is freed.
+ *
+ * If <b>make_old</b> is true, instead of deleting the router, we try adding
+ * it to rl-&gt;old_routers. */
+void
+routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, time_t now)
+{
+  routerinfo_t *ri_tmp;
+  extrainfo_t *ei_tmp;
+  int idx = ri->cache_info.routerlist_index;
+  tor_assert(0 <= idx && idx < smartlist_len(rl->routers));
+  tor_assert(smartlist_get(rl->routers, idx) == ri);
+
+  nodelist_remove_routerinfo(ri);
+
+  /* make sure the rephist module knows that it's not running */
+  rep_hist_note_router_unreachable(ri->cache_info.identity_digest, now);
+
+  ri->cache_info.routerlist_index = -1;
+  smartlist_del(rl->routers, idx);
+  if (idx < smartlist_len(rl->routers)) {
+    routerinfo_t *r = smartlist_get(rl->routers, idx);
+    r->cache_info.routerlist_index = idx;
+  }
+
+  ri_tmp = rimap_remove(rl->identity_map, ri->cache_info.identity_digest);
+  router_dir_info_changed();
+  tor_assert(ri_tmp == ri);
+
+  if (make_old && should_cache_old_descriptors() &&
+      ri->purpose == ROUTER_PURPOSE_GENERAL) {
+    signed_descriptor_t *sd;
+    sd = signed_descriptor_from_routerinfo(ri);
+    smartlist_add(rl->old_routers, sd);
+    sd->routerlist_index = smartlist_len(rl->old_routers)-1;
+    sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd);
+    if (!tor_digest_is_zero(sd->extra_info_digest))
+      sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd);
+  } else {
+    signed_descriptor_t *sd_tmp;
+    sd_tmp = sdmap_remove(rl->desc_digest_map,
+                          ri->cache_info.signed_descriptor_digest);
+    tor_assert(sd_tmp == &(ri->cache_info));
+    rl->desc_store.bytes_dropped += ri->cache_info.signed_descriptor_len;
+    ei_tmp = eimap_remove(rl->extra_info_map,
+                          ri->cache_info.extra_info_digest);
+    if (ei_tmp) {
+      rl->extrainfo_store.bytes_dropped +=
+        ei_tmp->cache_info.signed_descriptor_len;
+      extrainfo_free(ei_tmp);
+    }
+    if (!tor_digest_is_zero(ri->cache_info.extra_info_digest))
+      sdmap_remove(rl->desc_by_eid_map, ri->cache_info.extra_info_digest);
+    routerinfo_free(ri);
+  }
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Remove a signed_descriptor_t <b>sd</b> from <b>rl</b>-\>old_routers, and
+ * adjust <b>rl</b> as appropriate.  <b>idx</b> is -1, or the index of
+ * <b>sd</b>. */
+static void
+routerlist_remove_old(routerlist_t *rl, signed_descriptor_t *sd, int idx)
+{
+  signed_descriptor_t *sd_tmp;
+  extrainfo_t *ei_tmp;
+  desc_store_t *store;
+  if (idx == -1) {
+    idx = sd->routerlist_index;
+  }
+  tor_assert(0 <= idx && idx < smartlist_len(rl->old_routers));
+  /* XXXX edmanm's bridge relay triggered the following assert while
+   * running 0.2.0.12-alpha.  If anybody triggers this again, see if we
+   * can get a backtrace. */
+  tor_assert(smartlist_get(rl->old_routers, idx) == sd);
+  tor_assert(idx == sd->routerlist_index);
+
+  sd->routerlist_index = -1;
+  smartlist_del(rl->old_routers, idx);
+  if (idx < smartlist_len(rl->old_routers)) {
+    signed_descriptor_t *d = smartlist_get(rl->old_routers, idx);
+    d->routerlist_index = idx;
+  }
+  sd_tmp = sdmap_remove(rl->desc_digest_map,
+                        sd->signed_descriptor_digest);
+  tor_assert(sd_tmp == sd);
+  store = desc_get_store(rl, sd);
+  if (store)
+    store->bytes_dropped += sd->signed_descriptor_len;
+
+  ei_tmp = eimap_remove(rl->extra_info_map,
+                        sd->extra_info_digest);
+  if (ei_tmp) {
+    rl->extrainfo_store.bytes_dropped +=
+      ei_tmp->cache_info.signed_descriptor_len;
+    extrainfo_free(ei_tmp);
+  }
+  if (!tor_digest_is_zero(sd->extra_info_digest))
+    sdmap_remove(rl->desc_by_eid_map, sd->extra_info_digest);
+
+  signed_descriptor_free(sd);
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Remove <b>ri_old</b> from the routerlist <b>rl</b>, and replace it with
+ * <b>ri_new</b>, updating all index info.  If <b>idx</b> is nonnegative and
+ * smartlist_get(rl-&gt;routers, idx) == ri, we don't need to do a linear
+ * search over the list to decide which to remove.  We put ri_new in the same
+ * index as ri_old, if possible.  ri is freed as appropriate.
+ *
+ * If should_cache_descriptors() is true, instead of deleting the router,
+ * we add it to rl-&gt;old_routers. */
+static void
+routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old,
+                   routerinfo_t *ri_new)
+{
+  int idx;
+  int same_descriptors;
+
+  routerinfo_t *ri_tmp;
+  extrainfo_t *ei_tmp;
+  {
+    const routerinfo_t *ri_generated = router_get_my_routerinfo();
+    tor_assert(ri_generated != ri_new);
+  }
+  tor_assert(ri_old != ri_new);
+  tor_assert(ri_new->cache_info.routerlist_index == -1);
+
+  idx = ri_old->cache_info.routerlist_index;
+  tor_assert(0 <= idx && idx < smartlist_len(rl->routers));
+  tor_assert(smartlist_get(rl->routers, idx) == ri_old);
+
+  {
+    routerinfo_t *ri_old_tmp=NULL;
+    nodelist_set_routerinfo(ri_new, &ri_old_tmp);
+    tor_assert(ri_old == ri_old_tmp);
+  }
+
+  router_dir_info_changed();
+  if (idx >= 0) {
+    smartlist_set(rl->routers, idx, ri_new);
+    ri_old->cache_info.routerlist_index = -1;
+    ri_new->cache_info.routerlist_index = idx;
+    /* Check that ri_old is not in rl->routers anymore: */
+    tor_assert( routerlist_find_elt_(rl->routers, ri_old, -1) == -1 );
+  } else {
+    log_warn(LD_BUG, "Appending entry from routerlist_replace.");
+    routerlist_insert(rl, ri_new);
+    return;
+  }
+  if (tor_memneq(ri_old->cache_info.identity_digest,
+             ri_new->cache_info.identity_digest, DIGEST_LEN)) {
+    /* digests don't match; digestmap_set won't replace */
+    rimap_remove(rl->identity_map, ri_old->cache_info.identity_digest);
+  }
+  ri_tmp = rimap_set(rl->identity_map,
+                     ri_new->cache_info.identity_digest, ri_new);
+  tor_assert(!ri_tmp || ri_tmp == ri_old);
+  sdmap_set(rl->desc_digest_map,
+            ri_new->cache_info.signed_descriptor_digest,
+            &(ri_new->cache_info));
+
+  if (!tor_digest_is_zero(ri_new->cache_info.extra_info_digest)) {
+    sdmap_set(rl->desc_by_eid_map, ri_new->cache_info.extra_info_digest,
+              &ri_new->cache_info);
+  }
+
+  same_descriptors = tor_memeq(ri_old->cache_info.signed_descriptor_digest,
+                              ri_new->cache_info.signed_descriptor_digest,
+                              DIGEST_LEN);
+
+  if (should_cache_old_descriptors() &&
+      ri_old->purpose == ROUTER_PURPOSE_GENERAL &&
+      !same_descriptors) {
+    /* ri_old is going to become a signed_descriptor_t and go into
+     * old_routers */
+    signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri_old);
+    smartlist_add(rl->old_routers, sd);
+    sd->routerlist_index = smartlist_len(rl->old_routers)-1;
+    sdmap_set(rl->desc_digest_map, sd->signed_descriptor_digest, sd);
+    if (!tor_digest_is_zero(sd->extra_info_digest))
+      sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd);
+  } else {
+    /* We're dropping ri_old. */
+    if (!same_descriptors) {
+      /* digests don't match; The sdmap_set above didn't replace */
+      sdmap_remove(rl->desc_digest_map,
+                   ri_old->cache_info.signed_descriptor_digest);
+
+      if (tor_memneq(ri_old->cache_info.extra_info_digest,
+                 ri_new->cache_info.extra_info_digest, DIGEST_LEN)) {
+        ei_tmp = eimap_remove(rl->extra_info_map,
+                              ri_old->cache_info.extra_info_digest);
+        if (ei_tmp) {
+          rl->extrainfo_store.bytes_dropped +=
+            ei_tmp->cache_info.signed_descriptor_len;
+          extrainfo_free(ei_tmp);
+        }
+      }
+
+      if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) {
+        sdmap_remove(rl->desc_by_eid_map,
+                     ri_old->cache_info.extra_info_digest);
+      }
+    }
+    rl->desc_store.bytes_dropped += ri_old->cache_info.signed_descriptor_len;
+    routerinfo_free(ri_old);
+  }
+#ifdef DEBUG_ROUTERLIST
+  routerlist_assert_ok(rl);
+#endif
+}
+
+/** Extract the descriptor <b>sd</b> from old_routerlist, and re-parse
+ * it as a fresh routerinfo_t. */
+static routerinfo_t *
+routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd)
+{
+  routerinfo_t *ri;
+  const char *body;
+
+  body = signed_descriptor_get_annotations(sd);
+
+  ri = router_parse_entry_from_string(body,
+                         body+sd->signed_descriptor_len+sd->annotations_len,
+                         0, 1, NULL, NULL);
+  if (!ri)
+    return NULL;
+  signed_descriptor_move(&ri->cache_info, sd);
+
+  routerlist_remove_old(rl, sd, -1);
+
+  return ri;
+}
+
+/** Free all memory held by the routerlist module.
+ * Note: Calling routerlist_free_all() should always be paired with
+ * a call to nodelist_free_all(). These should only be called during
+ * cleanup.
+ */
+void
+routerlist_free_all(void)
+{
+  routerlist_free(routerlist);
+  routerlist = NULL;
+  if (warned_nicknames) {
+    SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp));
+    smartlist_free(warned_nicknames);
+    warned_nicknames = NULL;
+  }
+  clear_dir_servers();
+  smartlist_free(trusted_dir_servers);
+  smartlist_free(fallback_dir_servers);
+  trusted_dir_servers = fallback_dir_servers = NULL;
+  if (trusted_dir_certs) {
+    digestmap_free(trusted_dir_certs, cert_list_free_void);
+    trusted_dir_certs = NULL;
+  }
+}
+
+/** Forget that we have issued any router-related warnings, so that we'll
+ * warn again if we see the same errors. */
+void
+routerlist_reset_warnings(void)
+{
+  if (!warned_nicknames)
+    warned_nicknames = smartlist_new();
+  SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp));
+  smartlist_clear(warned_nicknames); /* now the list is empty. */
+
+  networkstatus_reset_warnings();
+}
+
+/** Return 1 if the signed descriptor of this router is older than
+ *  <b>seconds</b> seconds.  Otherwise return 0. */
+MOCK_IMPL(int,
+router_descriptor_is_older_than,(const routerinfo_t *router, int seconds))
+{
+  return router->cache_info.published_on < approx_time() - seconds;
+}
+
+/** Add <b>router</b> to the routerlist, if we don't already have it.  Replace
+ * older entries (if any) with the same key.  Note: Callers should not hold
+ * their pointers to <b>router</b> if this function fails; <b>router</b>
+ * will either be inserted into the routerlist or freed. Similarly, even
+ * if this call succeeds, they should not hold their pointers to
+ * <b>router</b> after subsequent calls with other routerinfo's -- they
+ * might cause the original routerinfo to get freed.
+ *
+ * Returns the status for the operation. Might set *<b>msg</b> if it wants
+ * the poster of the router to know something.
+ *
+ * If <b>from_cache</b>, this descriptor came from our disk cache. If
+ * <b>from_fetch</b>, we received it in response to a request we made.
+ * (If both are false, that means it was uploaded to us as an auth dir
+ * server or via the controller.)
+ *
+ * This function should be called *after*
+ * routers_update_status_from_consensus_networkstatus; subsequently, you
+ * should call router_rebuild_store and routerlist_descriptors_added.
+ */
+was_router_added_t
+router_add_to_routerlist(routerinfo_t *router, const char **msg,
+                         int from_cache, int from_fetch)
+{
+  const char *id_digest;
+  const or_options_t *options = get_options();
+  int authdir = authdir_mode_handles_descs(options, router->purpose);
+  int authdir_believes_valid = 0;
+  routerinfo_t *old_router;
+  networkstatus_t *consensus =
+    networkstatus_get_latest_consensus_by_flavor(FLAV_NS);
+  int in_consensus = 0;
+
+  tor_assert(msg);
+
+  if (!routerlist)
+    router_get_routerlist();
+
+  id_digest = router->cache_info.identity_digest;
+
+  old_router = router_get_mutable_by_digest(id_digest);
+
+  /* Make sure that it isn't expired. */
+  if (router->cert_expiration_time < approx_time()) {
+    routerinfo_free(router);
+    *msg = "Some certs on this router are expired.";
+    return ROUTER_CERTS_EXPIRED;
+  }
+
+  /* Make sure that we haven't already got this exact descriptor. */
+  if (sdmap_get(routerlist->desc_digest_map,
+                router->cache_info.signed_descriptor_digest)) {
+    /* If we have this descriptor already and the new descriptor is a bridge
+     * descriptor, replace it. If we had a bridge descriptor before and the
+     * new one is not a bridge descriptor, don't replace it. */
+
+    /* Only members of routerlist->identity_map can be bridges; we don't
+     * put bridges in old_routers. */
+    const int was_bridge = old_router &&
+      old_router->purpose == ROUTER_PURPOSE_BRIDGE;
+
+    if (routerinfo_is_a_configured_bridge(router) &&
+        router->purpose == ROUTER_PURPOSE_BRIDGE &&
+        !was_bridge) {
+      log_info(LD_DIR, "Replacing non-bridge descriptor with bridge "
+               "descriptor for router %s",
+               router_describe(router));
+    } else {
+      log_info(LD_DIR,
+               "Dropping descriptor that we already have for router %s",
+               router_describe(router));
+      *msg = "Router descriptor was not new.";
+      routerinfo_free(router);
+      return ROUTER_IS_ALREADY_KNOWN;
+    }
+  }
+
+  if (authdir) {
+    if (authdir_wants_to_reject_router(router, msg,
+                                       !from_cache && !from_fetch,
+                                       &authdir_believes_valid)) {
+      tor_assert(*msg);
+      routerinfo_free(router);
+      return ROUTER_AUTHDIR_REJECTS;
+    }
+  } else if (from_fetch) {
+    /* Only check the descriptor digest against the network statuses when
+     * we are receiving in response to a fetch. */
+
+    if (!signed_desc_digest_is_recognized(&router->cache_info) &&
+        !routerinfo_is_a_configured_bridge(router)) {
+      /* We asked for it, so some networkstatus must have listed it when we
+       * did.  Save it if we're a cache in case somebody else asks for it. */
+      log_info(LD_DIR,
+               "Received a no-longer-recognized descriptor for router %s",
+               router_describe(router));
+      *msg = "Router descriptor is not referenced by any network-status.";
+
+      /* Only journal this desc if we want to keep old descriptors */
+      if (!from_cache && should_cache_old_descriptors())
+        signed_desc_append_to_journal(&router->cache_info,
+                                      &routerlist->desc_store);
+      routerlist_insert_old(routerlist, router);
+      return ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS;
+    }
+  }
+
+  /* We no longer need a router with this descriptor digest. */
+  if (consensus) {
+    routerstatus_t *rs = networkstatus_vote_find_mutable_entry(
+                                                     consensus, id_digest);
+    if (rs && tor_memeq(rs->descriptor_digest,
+                      router->cache_info.signed_descriptor_digest,
+                      DIGEST_LEN)) {
+      in_consensus = 1;
+    }
+  }
+
+  if (router->purpose == ROUTER_PURPOSE_GENERAL &&
+      consensus && !in_consensus && !authdir) {
+    /* If it's a general router not listed in the consensus, then don't
+     * consider replacing the latest router with it. */
+    if (!from_cache && should_cache_old_descriptors())
+      signed_desc_append_to_journal(&router->cache_info,
+                                    &routerlist->desc_store);
+    routerlist_insert_old(routerlist, router);
+    *msg = "Skipping router descriptor: not in consensus.";
+    return ROUTER_NOT_IN_CONSENSUS;
+  }
+
+  /* If we're reading a bridge descriptor from our cache, and we don't
+   * recognize it as one of our currently configured bridges, drop the
+   * descriptor. Otherwise we could end up using it as one of our entry
+   * guards even if it isn't in our Bridge config lines. */
+  if (router->purpose == ROUTER_PURPOSE_BRIDGE && from_cache &&
+      !authdir_mode_bridge(options) &&
+      !routerinfo_is_a_configured_bridge(router)) {
+    log_info(LD_DIR, "Dropping bridge descriptor for %s because we have "
+             "no bridge configured at that address.",
+             safe_str_client(router_describe(router)));
+    *msg = "Router descriptor was not a configured bridge.";
+    routerinfo_free(router);
+    return ROUTER_WAS_NOT_WANTED;
+  }
+
+  /* If we have a router with the same identity key, choose the newer one. */
+  if (old_router) {
+    if (!in_consensus && (router->cache_info.published_on <=
+                          old_router->cache_info.published_on)) {
+      /* Same key, but old.  This one is not listed in the consensus. */
+      log_debug(LD_DIR, "Not-new descriptor for router %s",
+                router_describe(router));
+      /* Only journal this desc if we'll be serving it. */
+      if (!from_cache && should_cache_old_descriptors())
+        signed_desc_append_to_journal(&router->cache_info,
+                                      &routerlist->desc_store);
+      routerlist_insert_old(routerlist, router);
+      *msg = "Router descriptor was not new.";
+      return ROUTER_IS_ALREADY_KNOWN;
+    } else {
+      /* Same key, and either new, or listed in the consensus. */
+      log_debug(LD_DIR, "Replacing entry for router %s",
+                router_describe(router));
+      routerlist_replace(routerlist, old_router, router);
+      if (!from_cache) {
+        signed_desc_append_to_journal(&router->cache_info,
+                                      &routerlist->desc_store);
+      }
+      *msg = authdir_believes_valid ? "Valid server updated" :
+        ("Invalid server updated. (This dirserver is marking your "
+         "server as unapproved.)");
+      return ROUTER_ADDED_SUCCESSFULLY;
+    }
+  }
+
+  if (!in_consensus && from_cache &&
+      router_descriptor_is_older_than(router, OLD_ROUTER_DESC_MAX_AGE)) {
+    *msg = "Router descriptor was really old.";
+    routerinfo_free(router);
+    return ROUTER_WAS_TOO_OLD;
+  }
+
+  /* We haven't seen a router with this identity before. Add it to the end of
+   * the list. */
+  routerlist_insert(routerlist, router);
+  if (!from_cache) {
+    signed_desc_append_to_journal(&router->cache_info,
+                                  &routerlist->desc_store);
+  }
+  return ROUTER_ADDED_SUCCESSFULLY;
+}
+
+/** Insert <b>ei</b> into the routerlist, or free it. Other arguments are
+ * as for router_add_to_routerlist().  Return ROUTER_ADDED_SUCCESSFULLY iff
+ * we actually inserted it, ROUTER_BAD_EI otherwise.
+ */
+was_router_added_t
+router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg,
+                                   int from_cache, int from_fetch)
+{
+  was_router_added_t inserted;
+  (void)from_fetch;
+  if (msg) *msg = NULL;
+  /*XXXX Do something with msg */
+
+  inserted = extrainfo_insert(router_get_routerlist(), ei, !from_cache);
+
+  if (WRA_WAS_ADDED(inserted) && !from_cache)
+    signed_desc_append_to_journal(&ei->cache_info,
+                                  &routerlist->extrainfo_store);
+
+  return inserted;
+}
+
+/** Sorting helper: return &lt;0, 0, or &gt;0 depending on whether the
+ * signed_descriptor_t* in *<b>a</b> has an identity digest preceding, equal
+ * to, or later than that of *<b>b</b>. */
+static int
+compare_old_routers_by_identity_(const void **_a, const void **_b)
+{
+  int i;
+  const signed_descriptor_t *r1 = *_a, *r2 = *_b;
+  if ((i = fast_memcmp(r1->identity_digest, r2->identity_digest, DIGEST_LEN)))
+    return i;
+  return (int)(r1->published_on - r2->published_on);
+}
+
+/** Internal type used to represent how long an old descriptor was valid,
+ * where it appeared in the list of old descriptors, and whether it's extra
+ * old. Used only by routerlist_remove_old_cached_routers_with_id(). */
+struct duration_idx_t {
+  int duration;
+  int idx;
+  int old;
+};
+
+/** Sorting helper: compare two duration_idx_t by their duration. */
+static int
+compare_duration_idx_(const void *_d1, const void *_d2)
+{
+  const struct duration_idx_t *d1 = _d1;
+  const struct duration_idx_t *d2 = _d2;
+  return d1->duration - d2->duration;
+}
+
+/** The range <b>lo</b> through <b>hi</b> inclusive of routerlist->old_routers
+ * must contain routerinfo_t with the same identity and with publication time
+ * in ascending order.  Remove members from this range until there are no more
+ * than max_descriptors_per_router() remaining.  Start by removing the oldest
+ * members from before <b>cutoff</b>, then remove members which were current
+ * for the lowest amount of time.  The order of members of old_routers at
+ * indices <b>lo</b> or higher may be changed.
+ */
+static void
+routerlist_remove_old_cached_routers_with_id(time_t now,
+                                             time_t cutoff, int lo, int hi,
+                                             digestset_t *retain)
+{
+  int i, n = hi-lo+1;
+  unsigned n_extra, n_rmv = 0;
+  struct duration_idx_t *lifespans;
+  uint8_t *rmv, *must_keep;
+  smartlist_t *lst = routerlist->old_routers;
+#if 1
+  const char *ident;
+  tor_assert(hi < smartlist_len(lst));
+  tor_assert(lo <= hi);
+  ident = ((signed_descriptor_t*)smartlist_get(lst, lo))->identity_digest;
+  for (i = lo+1; i <= hi; ++i) {
+    signed_descriptor_t *r = smartlist_get(lst, i);
+    tor_assert(tor_memeq(ident, r->identity_digest, DIGEST_LEN));
+  }
+#endif /* 1 */
+  /* Check whether we need to do anything at all. */
+  {
+    int mdpr = directory_caches_dir_info(get_options()) ? 2 : 1;
+    if (n <= mdpr)
+      return;
+    n_extra = n - mdpr;
+  }
+
+  lifespans = tor_calloc(n, sizeof(struct duration_idx_t));
+  rmv = tor_calloc(n, sizeof(uint8_t));
+  must_keep = tor_calloc(n, sizeof(uint8_t));
+  /* Set lifespans to contain the lifespan and index of each server. */
+  /* Set rmv[i-lo]=1 if we're going to remove a server for being too old. */
+  for (i = lo; i <= hi; ++i) {
+    signed_descriptor_t *r = smartlist_get(lst, i);
+    signed_descriptor_t *r_next;
+    lifespans[i-lo].idx = i;
+    if (r->last_listed_as_valid_until >= now ||
+        (retain && digestset_probably_contains(retain,
+                                               r->signed_descriptor_digest))) {
+      must_keep[i-lo] = 1;
+    }
+    if (i < hi) {
+      r_next = smartlist_get(lst, i+1);
+      tor_assert(r->published_on <= r_next->published_on);
+      lifespans[i-lo].duration = (int)(r_next->published_on - r->published_on);
+    } else {
+      r_next = NULL;
+      lifespans[i-lo].duration = INT_MAX;
+    }
+    if (!must_keep[i-lo] && r->published_on < cutoff && n_rmv < n_extra) {
+      ++n_rmv;
+      lifespans[i-lo].old = 1;
+      rmv[i-lo] = 1;
+    }
+  }
+
+  if (n_rmv < n_extra) {
+    /**
+     * We aren't removing enough servers for being old.  Sort lifespans by
+     * the duration of liveness, and remove the ones we're not already going to
+     * remove based on how long they were alive.
+     **/
+    qsort(lifespans, n, sizeof(struct duration_idx_t), compare_duration_idx_);
+    for (i = 0; i < n && n_rmv < n_extra; ++i) {
+      if (!must_keep[lifespans[i].idx-lo] && !lifespans[i].old) {
+        rmv[lifespans[i].idx-lo] = 1;
+        ++n_rmv;
+      }
+    }
+  }
+
+  i = hi;
+  do {
+    if (rmv[i-lo])
+      routerlist_remove_old(routerlist, smartlist_get(lst, i), i);
+  } while (--i >= lo);
+  tor_free(must_keep);
+  tor_free(rmv);
+  tor_free(lifespans);
+}
+
+/** Deactivate any routers from the routerlist that are more than
+ * ROUTER_MAX_AGE seconds old and not recommended by any networkstatuses;
+ * remove old routers from the list of cached routers if we have too many.
+ */
+void
+routerlist_remove_old_routers(void)
+{
+  int i, hi=-1;
+  const char *cur_id = NULL;
+  time_t now = time(NULL);
+  time_t cutoff;
+  routerinfo_t *router;
+  signed_descriptor_t *sd;
+  digestset_t *retain;
+  const networkstatus_t *consensus = networkstatus_get_latest_consensus();
+
+  trusted_dirs_remove_old_certs();
+
+  if (!routerlist || !consensus)
+    return;
+
+  // routerlist_assert_ok(routerlist);
+
+  /* We need to guess how many router descriptors we will wind up wanting to
+     retain, so that we can be sure to allocate a large enough Bloom filter
+     to hold the digest set.  Overestimating is fine; underestimating is bad.
+  */
+  {
+    /* We'll probably retain everything in the consensus. */
+    int n_max_retain = smartlist_len(consensus->routerstatus_list);
+    retain = digestset_new(n_max_retain);
+  }
+
+  cutoff = now - OLD_ROUTER_DESC_MAX_AGE;
+  /* Retain anything listed in the consensus. */
+  if (consensus) {
+    SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs,
+        if (rs->published_on >= cutoff)
+          digestset_add(retain, rs->descriptor_digest));
+  }
+
+  /* If we have a consensus, we should consider pruning current routers that
+   * are too old and that nobody recommends.  (If we don't have a consensus,
+   * then we should get one before we decide to kill routers.) */
+
+  if (consensus) {
+    cutoff = now - ROUTER_MAX_AGE;
+    /* Remove too-old unrecommended members of routerlist->routers. */
+    for (i = 0; i < smartlist_len(routerlist->routers); ++i) {
+      router = smartlist_get(routerlist->routers, i);
+      if (router->cache_info.published_on <= cutoff &&
+          router->cache_info.last_listed_as_valid_until < now &&
+          !digestset_probably_contains(retain,
+                          router->cache_info.signed_descriptor_digest)) {
+        /* Too old: remove it.  (If we're a cache, just move it into
+         * old_routers.) */
+        log_info(LD_DIR,
+                 "Forgetting obsolete (too old) routerinfo for router %s",
+                 router_describe(router));
+        routerlist_remove(routerlist, router, 1, now);
+        i--;
+      }
+    }
+  }
+
+  //routerlist_assert_ok(routerlist);
+
+  /* Remove far-too-old members of routerlist->old_routers. */
+  cutoff = now - OLD_ROUTER_DESC_MAX_AGE;
+  for (i = 0; i < smartlist_len(routerlist->old_routers); ++i) {
+    sd = smartlist_get(routerlist->old_routers, i);
+    if (sd->published_on <= cutoff &&
+        sd->last_listed_as_valid_until < now &&
+        !digestset_probably_contains(retain, sd->signed_descriptor_digest)) {
+      /* Too old.  Remove it. */
+      routerlist_remove_old(routerlist, sd, i--);
+    }
+  }
+
+  //routerlist_assert_ok(routerlist);
+
+  log_info(LD_DIR, "We have %d live routers and %d old router descriptors.",
+           smartlist_len(routerlist->routers),
+           smartlist_len(routerlist->old_routers));
+
+  /* Now we might have to look at routerlist->old_routers for extraneous
+   * members. (We'd keep all the members if we could, but we need to save
+   * space.) First, check whether we have too many router descriptors, total.
+   * We're okay with having too many for some given router, so long as the
+   * total number doesn't approach max_descriptors_per_router()*len(router).
+   */
+  if (smartlist_len(routerlist->old_routers) <
+      smartlist_len(routerlist->routers))
+    goto done;
+
+  /* Sort by identity, then fix indices. */
+  smartlist_sort(routerlist->old_routers, compare_old_routers_by_identity_);
+  /* Fix indices. */
+  for (i = 0; i < smartlist_len(routerlist->old_routers); ++i) {
+    signed_descriptor_t *r = smartlist_get(routerlist->old_routers, i);
+    r->routerlist_index = i;
+  }
+
+  /* Iterate through the list from back to front, so when we remove descriptors
+   * we don't mess up groups we haven't gotten to. */
+  for (i = smartlist_len(routerlist->old_routers)-1; i >= 0; --i) {
+    signed_descriptor_t *r = smartlist_get(routerlist->old_routers, i);
+    if (!cur_id) {
+      cur_id = r->identity_digest;
+      hi = i;
+    }
+    if (tor_memneq(cur_id, r->identity_digest, DIGEST_LEN)) {
+      routerlist_remove_old_cached_routers_with_id(now,
+                                                   cutoff, i+1, hi, retain);
+      cur_id = r->identity_digest;
+      hi = i;
+    }
+  }
+  if (hi>=0)
+    routerlist_remove_old_cached_routers_with_id(now, cutoff, 0, hi, retain);
+  //routerlist_assert_ok(routerlist);
+
+ done:
+  digestset_free(retain);
+  router_rebuild_store(RRS_DONT_REMOVE_OLD, &routerlist->desc_store);
+  router_rebuild_store(RRS_DONT_REMOVE_OLD,&routerlist->extrainfo_store);
+}
+
+/** We just added a new set of descriptors. Take whatever extra steps
+ * we need. */
+void
+routerlist_descriptors_added(smartlist_t *sl, int from_cache)
+{
+  tor_assert(sl);
+  control_event_descriptors_changed(sl);
+  SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) {
+    if (ri->purpose == ROUTER_PURPOSE_BRIDGE)
+      learned_bridge_descriptor(ri, from_cache);
+    if (ri->needs_retest_if_added) {
+      ri->needs_retest_if_added = 0;
+      dirserv_single_reachability_test(approx_time(), ri);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+}
+
+/**
+ * Code to parse a single router descriptor and insert it into the
+ * routerlist.  Return -1 if the descriptor was ill-formed; 0 if the
+ * descriptor was well-formed but could not be added; and 1 if the
+ * descriptor was added.
+ *
+ * If we don't add it and <b>msg</b> is not NULL, then assign to
+ * *<b>msg</b> a static string describing the reason for refusing the
+ * descriptor.
+ *
+ * This is used only by the controller.
+ */
+int
+router_load_single_router(const char *s, uint8_t purpose, int cache,
+                          const char **msg)
+{
+  routerinfo_t *ri;
+  was_router_added_t r;
+  smartlist_t *lst;
+  char annotation_buf[ROUTER_ANNOTATION_BUF_LEN];
+  tor_assert(msg);
+  *msg = NULL;
+
+  tor_snprintf(annotation_buf, sizeof(annotation_buf),
+               "@source controller\n"
+               "@purpose %s\n", router_purpose_to_string(purpose));
+
+  if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0,
+                                            annotation_buf, NULL))) {
+    log_warn(LD_DIR, "Error parsing router descriptor; dropping.");
+    *msg = "Couldn't parse router descriptor.";
+    return -1;
+  }
+  tor_assert(ri->purpose == purpose);
+  if (router_is_me(ri)) {
+    log_warn(LD_DIR, "Router's identity key matches mine; dropping.");
+    *msg = "Router's identity key matches mine.";
+    routerinfo_free(ri);
+    return 0;
+  }
+
+  if (!cache) /* obey the preference of the controller */
+    ri->cache_info.do_not_cache = 1;
+
+  lst = smartlist_new();
+  smartlist_add(lst, ri);
+  routers_update_status_from_consensus_networkstatus(lst, 0);
+
+  r = router_add_to_routerlist(ri, msg, 0, 0);
+  if (!WRA_WAS_ADDED(r)) {
+    /* we've already assigned to *msg now, and ri is already freed */
+    tor_assert(*msg);
+    if (r == ROUTER_AUTHDIR_REJECTS)
+      log_warn(LD_DIR, "Couldn't add router to list: %s Dropping.", *msg);
+    smartlist_free(lst);
+    return 0;
+  } else {
+    routerlist_descriptors_added(lst, 0);
+    smartlist_free(lst);
+    log_debug(LD_DIR, "Added router to list");
+    return 1;
+  }
+}
+
+/** Given a string <b>s</b> containing some routerdescs, parse it and put the
+ * routers into our directory.  If saved_location is SAVED_NOWHERE, the routers
+ * are in response to a query to the network: cache them by adding them to
+ * the journal.
+ *
+ * Return the number of routers actually added.
+ *
+ * If <b>requested_fingerprints</b> is provided, it must contain a list of
+ * uppercased fingerprints.  Do not update any router whose
+ * fingerprint is not on the list; after updating a router, remove its
+ * fingerprint from the list.
+ *
+ * If <b>descriptor_digests</b> is non-zero, then the requested_fingerprints
+ * are descriptor digests. Otherwise they are identity digests.
+ */
+int
+router_load_routers_from_string(const char *s, const char *eos,
+                                saved_location_t saved_location,
+                                smartlist_t *requested_fingerprints,
+                                int descriptor_digests,
+                                const char *prepend_annotations)
+{
+  smartlist_t *routers = smartlist_new(), *changed = smartlist_new();
+  char fp[HEX_DIGEST_LEN+1];
+  const char *msg;
+  int from_cache = (saved_location != SAVED_NOWHERE);
+  int allow_annotations = (saved_location != SAVED_NOWHERE);
+  int any_changed = 0;
+  smartlist_t *invalid_digests = smartlist_new();
+
+  router_parse_list_from_string(&s, eos, routers, saved_location, 0,
+                                allow_annotations, prepend_annotations,
+                                invalid_digests);
+
+  routers_update_status_from_consensus_networkstatus(routers, !from_cache);
+
+  log_info(LD_DIR, "%d elements to add", smartlist_len(routers));
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    was_router_added_t r;
+    char d[DIGEST_LEN];
+    if (requested_fingerprints) {
+      base16_encode(fp, sizeof(fp), descriptor_digests ?
+                      ri->cache_info.signed_descriptor_digest :
+                      ri->cache_info.identity_digest,
+                    DIGEST_LEN);
+      if (smartlist_contains_string(requested_fingerprints, fp)) {
+        smartlist_string_remove(requested_fingerprints, fp);
+      } else {
+        char *requested =
+          smartlist_join_strings(requested_fingerprints," ",0,NULL);
+        log_warn(LD_DIR,
+                 "We received a router descriptor with a fingerprint (%s) "
+                 "that we never requested. (We asked for: %s.) Dropping.",
+                 fp, requested);
+        tor_free(requested);
+        routerinfo_free(ri);
+        continue;
+      }
+    }
+
+    memcpy(d, ri->cache_info.signed_descriptor_digest, DIGEST_LEN);
+    r = router_add_to_routerlist(ri, &msg, from_cache, !from_cache);
+    if (WRA_WAS_ADDED(r)) {
+      any_changed++;
+      smartlist_add(changed, ri);
+      routerlist_descriptors_added(changed, from_cache);
+      smartlist_clear(changed);
+    } else if (WRA_NEVER_DOWNLOADABLE(r)) {
+      download_status_t *dl_status;
+      dl_status = router_get_dl_status_by_descriptor_digest(d);
+      if (dl_status) {
+        log_info(LD_GENERAL, "Marking router %s as never downloadable",
+                 hex_str(d, DIGEST_LEN));
+        download_status_mark_impossible(dl_status);
+      }
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) {
+    /* This digest is never going to be parseable. */
+    base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN);
+    if (requested_fingerprints && descriptor_digests) {
+      if (! smartlist_contains_string(requested_fingerprints, fp)) {
+        /* But we didn't ask for it, so we should assume shennanegans. */
+        continue;
+      }
+      smartlist_string_remove(requested_fingerprints, fp);
+    }
+    download_status_t *dls;
+    dls = router_get_dl_status_by_descriptor_digest((char*)bad_digest);
+    if (dls) {
+      log_info(LD_GENERAL, "Marking router with descriptor %s as unparseable, "
+               "and therefore undownloadable", fp);
+      download_status_mark_impossible(dls);
+    }
+  } SMARTLIST_FOREACH_END(bad_digest);
+  SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
+  smartlist_free(invalid_digests);
+
+  routerlist_assert_ok(routerlist);
+
+  if (any_changed)
+    router_rebuild_store(0, &routerlist->desc_store);
+
+  smartlist_free(routers);
+  smartlist_free(changed);
+
+  return any_changed;
+}
+
+/** Parse one or more extrainfos from <b>s</b> (ending immediately before
+ * <b>eos</b> if <b>eos</b> is present).  Other arguments are as for
+ * router_load_routers_from_string(). */
+void
+router_load_extrainfo_from_string(const char *s, const char *eos,
+                                  saved_location_t saved_location,
+                                  smartlist_t *requested_fingerprints,
+                                  int descriptor_digests)
+{
+  smartlist_t *extrainfo_list = smartlist_new();
+  const char *msg;
+  int from_cache = (saved_location != SAVED_NOWHERE);
+  smartlist_t *invalid_digests = smartlist_new();
+
+  router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1, 0,
+                                NULL, invalid_digests);
+
+  log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list));
+
+  SMARTLIST_FOREACH_BEGIN(extrainfo_list, extrainfo_t *, ei) {
+      uint8_t d[DIGEST_LEN];
+      memcpy(d, ei->cache_info.signed_descriptor_digest, DIGEST_LEN);
+      was_router_added_t added =
+        router_add_extrainfo_to_routerlist(ei, &msg, from_cache, !from_cache);
+      if (WRA_WAS_ADDED(added) && requested_fingerprints) {
+        char fp[HEX_DIGEST_LEN+1];
+        base16_encode(fp, sizeof(fp), descriptor_digests ?
+                        ei->cache_info.signed_descriptor_digest :
+                        ei->cache_info.identity_digest,
+                      DIGEST_LEN);
+        smartlist_string_remove(requested_fingerprints, fp);
+        /* We silently let relays stuff us with extrainfos we didn't ask for,
+         * so long as we would have wanted them anyway.  Since we always fetch
+         * all the extrainfos we want, and we never actually act on them
+         * inside Tor, this should be harmless. */
+      } else if (WRA_NEVER_DOWNLOADABLE(added)) {
+        signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d);
+        if (sd) {
+          log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as "
+                   "unparseable, and therefore undownloadable",
+                   hex_str((char*)d,DIGEST_LEN));
+          download_status_mark_impossible(&sd->ei_dl_status);
+        }
+      }
+  } SMARTLIST_FOREACH_END(ei);
+
+  SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) {
+    /* This digest is never going to be parseable. */
+    char fp[HEX_DIGEST_LEN+1];
+    base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN);
+    if (requested_fingerprints) {
+      if (! smartlist_contains_string(requested_fingerprints, fp)) {
+        /* But we didn't ask for it, so we should assume shennanegans. */
+        continue;
+      }
+      smartlist_string_remove(requested_fingerprints, fp);
+    }
+    signed_descriptor_t *sd =
+      router_get_by_extrainfo_digest((char*)bad_digest);
+    if (sd) {
+      log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as "
+               "unparseable, and therefore undownloadable", fp);
+      download_status_mark_impossible(&sd->ei_dl_status);
+    }
+  } SMARTLIST_FOREACH_END(bad_digest);
+  SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
+  smartlist_free(invalid_digests);
+
+  routerlist_assert_ok(routerlist);
+  router_rebuild_store(0, &router_get_routerlist()->extrainfo_store);
+
+  smartlist_free(extrainfo_list);
+}
+
+/** Return true iff the latest ns-flavored consensus includes a descriptor
+ * whose digest is that of <b>desc</b>. */
+static int
+signed_desc_digest_is_recognized(signed_descriptor_t *desc)
+{
+  const routerstatus_t *rs;
+  networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(
+                                                                      FLAV_NS);
+
+  if (consensus) {
+    rs = networkstatus_vote_find_entry(consensus, desc->identity_digest);
+    if (rs && tor_memeq(rs->descriptor_digest,
+                      desc->signed_descriptor_digest, DIGEST_LEN))
+      return 1;
+  }
+  return 0;
+}
+
+/** Update downloads for router descriptors and/or microdescriptors as
+ * appropriate. */
+void
+update_all_descriptor_downloads(time_t now)
+{
+  if (should_delay_dir_fetches(get_options(), NULL))
+    return;
+  update_router_descriptor_downloads(now);
+  update_microdesc_downloads(now);
+  launch_dummy_descriptor_download_as_needed(now, get_options());
+}
+
+/** Clear all our timeouts for fetching v3 directory stuff, and then
+ * give it all a try again. */
+void
+routerlist_retry_directory_downloads(time_t now)
+{
+  (void)now;
+
+  log_debug(LD_GENERAL,
+            "In routerlist_retry_directory_downloads()");
+
+  router_reset_status_download_failures();
+  router_reset_descriptor_download_failures();
+  reschedule_directory_downloads();
+}
+
+/** Return true iff <b>router</b> does not permit exit streams.
+ */
+int
+router_exit_policy_rejects_all(const routerinfo_t *router)
+{
+  return router->policy_is_reject_star;
+}
+
+/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity
+ * key <b>digest</b> which has DIGEST_LEN bytes.  If <b>address</b> is NULL,
+ * add ourself.  If <b>is_authority</b>, this is a directory authority.  Return
+ * the new directory server entry on success or NULL on failure. */
+static dir_server_t *
+dir_server_new(int is_authority,
+               const char *nickname,
+               const tor_addr_t *addr,
+               const char *hostname,
+               uint16_t dir_port, uint16_t or_port,
+               const tor_addr_port_t *addrport_ipv6,
+               const char *digest, const char *v3_auth_digest,
+               dirinfo_type_t type,
+               double weight)
+{
+  dir_server_t *ent;
+  uint32_t a;
+  char *hostname_ = NULL;
+
+  tor_assert(digest);
+
+  if (weight < 0)
+    return NULL;
+
+  if (tor_addr_family(addr) == AF_INET)
+    a = tor_addr_to_ipv4h(addr);
+  else
+    return NULL;
+
+  if (!hostname)
+    hostname_ = tor_addr_to_str_dup(addr);
+  else
+    hostname_ = tor_strdup(hostname);
+
+  ent = tor_malloc_zero(sizeof(dir_server_t));
+  ent->nickname = nickname ? tor_strdup(nickname) : NULL;
+  ent->address = hostname_;
+  ent->addr = a;
+  ent->dir_port = dir_port;
+  ent->or_port = or_port;
+  ent->is_running = 1;
+  ent->is_authority = is_authority;
+  ent->type = type;
+  ent->weight = weight;
+  if (addrport_ipv6) {
+    if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) {
+      log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6.");
+      tor_addr_make_unspec(&ent->ipv6_addr);
+    } else {
+      tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr);
+      ent->ipv6_orport = addrport_ipv6->port;
+    }
+  } else {
+    tor_addr_make_unspec(&ent->ipv6_addr);
+  }
+
+  memcpy(ent->digest, digest, DIGEST_LEN);
+  if (v3_auth_digest && (type & V3_DIRINFO))
+    memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN);
+
+  if (nickname)
+    tor_asprintf(&ent->description, "directory server \"%s\" at %s:%d",
+                 nickname, hostname_, (int)dir_port);
+  else
+    tor_asprintf(&ent->description, "directory server at %s:%d",
+                 hostname_, (int)dir_port);
+
+  ent->fake_status.addr = ent->addr;
+  tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr);
+  memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN);
+  if (nickname)
+    strlcpy(ent->fake_status.nickname, nickname,
+            sizeof(ent->fake_status.nickname));
+  else
+    ent->fake_status.nickname[0] = '\0';
+  ent->fake_status.dir_port = ent->dir_port;
+  ent->fake_status.or_port = ent->or_port;
+  ent->fake_status.ipv6_orport = ent->ipv6_orport;
+
+  return ent;
+}
+
+/** Create an authoritative directory server at
+ * <b>address</b>:<b>port</b>, with identity key <b>digest</b>.  If
+ * <b>address</b> is NULL, add ourself.  Return the new trusted directory
+ * server entry on success or NULL if we couldn't add it. */
+dir_server_t *
+trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const tor_addr_port_t *ipv6_addrport,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight)
+{
+  uint32_t a;
+  tor_addr_t addr;
+  char *hostname=NULL;
+  dir_server_t *result;
+
+  if (!address) { /* The address is us; we should guess. */
+    if (resolve_my_address(LOG_WARN, get_options(),
+                           &a, NULL, &hostname) < 0) {
+      log_warn(LD_CONFIG,
+               "Couldn't find a suitable address when adding ourself as a "
+               "trusted directory server.");
+      return NULL;
+    }
+    if (!hostname)
+      hostname = tor_dup_ip(a);
+  } else {
+    if (tor_lookup_hostname(address, &a)) {
+      log_warn(LD_CONFIG,
+               "Unable to lookup address for directory server at '%s'",
+               address);
+      return NULL;
+    }
+    hostname = tor_strdup(address);
+  }
+  tor_addr_from_ipv4h(&addr, a);
+
+  result = dir_server_new(1, nickname, &addr, hostname,
+                          dir_port, or_port,
+                          ipv6_addrport,
+                          digest,
+                          v3_auth_digest, type, weight);
+  tor_free(hostname);
+  return result;
+}
+
+/** Return a new dir_server_t for a fallback directory server at
+ * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest
+ * <b>id_digest</b> */
+dir_server_t *
+fallback_dir_server_new(const tor_addr_t *addr,
+                        uint16_t dir_port, uint16_t or_port,
+                        const tor_addr_port_t *addrport_ipv6,
+                        const char *id_digest, double weight)
+{
+  return dir_server_new(0, NULL, addr, NULL, dir_port, or_port,
+                        addrport_ipv6,
+                        id_digest,
+                        NULL, ALL_DIRINFO, weight);
+}
+
+/** Add a directory server to the global list(s). */
+void
+dir_server_add(dir_server_t *ent)
+{
+  if (!trusted_dir_servers)
+    trusted_dir_servers = smartlist_new();
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  if (ent->is_authority)
+    smartlist_add(trusted_dir_servers, ent);
+
+  smartlist_add(fallback_dir_servers, ent);
+  router_dir_info_changed();
+}
+
+/** Free storage held in <b>cert</b>. */
+void
+authority_cert_free_(authority_cert_t *cert)
+{
+  if (!cert)
+    return;
+
+  tor_free(cert->cache_info.signed_descriptor_body);
+  crypto_pk_free(cert->signing_key);
+  crypto_pk_free(cert->identity_key);
+
+  tor_free(cert);
+}
+
+#define dir_server_free(val) \
+  FREE_AND_NULL(dir_server_t, dir_server_free_, (val))
+
+/** Free storage held in <b>ds</b>. */
+static void
+dir_server_free_(dir_server_t *ds)
+{
+  if (!ds)
+    return;
+
+  tor_free(ds->nickname);
+  tor_free(ds->description);
+  tor_free(ds->address);
+  tor_free(ds);
+}
+
+/** Remove all members from the list of dir servers. */
+void
+clear_dir_servers(void)
+{
+  if (fallback_dir_servers) {
+    SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent,
+                      dir_server_free(ent));
+    smartlist_clear(fallback_dir_servers);
+  } else {
+    fallback_dir_servers = smartlist_new();
+  }
+  if (trusted_dir_servers) {
+    smartlist_clear(trusted_dir_servers);
+  } else {
+    trusted_dir_servers = smartlist_new();
+  }
+  router_dir_info_changed();
+}
+
+/** For every current directory connection whose purpose is <b>purpose</b>,
+ * and where the resource being downloaded begins with <b>prefix</b>, split
+ * rest of the resource into base16 fingerprints (or base64 fingerprints if
+ * purpose==DIR_PURPOSE_FETCH_MICRODESC), decode them, and set the
+ * corresponding elements of <b>result</b> to a nonzero value.
+ */
+static void
+list_pending_downloads(digestmap_t *result, digest256map_t *result256,
+                       int purpose, const char *prefix)
+{
+  const size_t p_len = strlen(prefix);
+  smartlist_t *tmp = smartlist_new();
+  smartlist_t *conns = get_connection_array();
+  int flags = DSR_HEX;
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC)
+    flags = DSR_DIGEST256|DSR_BASE64;
+
+  tor_assert(result || result256);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_DIR &&
+        conn->purpose == purpose &&
+        !conn->marked_for_close) {
+      const char *resource = TO_DIR_CONN(conn)->requested_resource;
+      if (!strcmpstart(resource, prefix))
+        dir_split_resource_into_fingerprints(resource + p_len,
+                                             tmp, NULL, flags);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  if (result) {
+    SMARTLIST_FOREACH(tmp, char *, d,
+                    {
+                      digestmap_set(result, d, (void*)1);
+                      tor_free(d);
+                    });
+  } else if (result256) {
+    SMARTLIST_FOREACH(tmp, uint8_t *, d,
+                    {
+                      digest256map_set(result256, d, (void*)1);
+                      tor_free(d);
+                    });
+  }
+  smartlist_free(tmp);
+}
+
+/** For every router descriptor (or extra-info document if <b>extrainfo</b> is
+ * true) we are currently downloading by descriptor digest, set result[d] to
+ * (void*)1. */
+static void
+list_pending_descriptor_downloads(digestmap_t *result, int extrainfo)
+{
+  int purpose =
+    extrainfo ? DIR_PURPOSE_FETCH_EXTRAINFO : DIR_PURPOSE_FETCH_SERVERDESC;
+  list_pending_downloads(result, NULL, purpose, "d/");
+}
+
+/** For every microdescriptor we are currently downloading by descriptor
+ * digest, set result[d] to (void*)1.
+ */
+void
+list_pending_microdesc_downloads(digest256map_t *result)
+{
+  list_pending_downloads(NULL, result, DIR_PURPOSE_FETCH_MICRODESC, "d/");
+}
+
+/** For every certificate we are currently downloading by (identity digest,
+ * signing key digest) pair, set result[fp_pair] to (void *1).
+ */
+static void
+list_pending_fpsk_downloads(fp_pair_map_t *result)
+{
+  const char *pfx = "fp-sk/";
+  smartlist_t *tmp;
+  smartlist_t *conns;
+  const char *resource;
+
+  tor_assert(result);
+
+  tmp = smartlist_new();
+  conns = get_connection_array();
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_DIR &&
+        conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE &&
+        !conn->marked_for_close) {
+      resource = TO_DIR_CONN(conn)->requested_resource;
+      if (!strcmpstart(resource, pfx))
+        dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx),
+                                                  tmp);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) {
+    fp_pair_map_set(result, fp, (void*)1);
+    tor_free(fp);
+  } SMARTLIST_FOREACH_END(fp);
+
+  smartlist_free(tmp);
+}
+
+/** Launch downloads for all the descriptors whose digests or digests256
+ * are listed as digests[i] for lo <= i < hi.  (Lo and hi may be out of
+ * range.)  If <b>source</b> is given, download from <b>source</b>;
+ * otherwise, download from an appropriate random directory server.
+ */
+MOCK_IMPL(STATIC void,
+initiate_descriptor_downloads,(const routerstatus_t *source,
+                               int purpose, smartlist_t *digests,
+                               int lo, int hi, int pds_flags))
+{
+  char *resource, *cp;
+  int digest_len, enc_digest_len;
+  const char *sep;
+  int b64_256;
+  smartlist_t *tmp;
+
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    /* Microdescriptors are downloaded by "-"-separated base64-encoded
+     * 256-bit digests. */
+    digest_len = DIGEST256_LEN;
+    enc_digest_len = BASE64_DIGEST256_LEN + 1;
+    sep = "-";
+    b64_256 = 1;
+  } else {
+    digest_len = DIGEST_LEN;
+    enc_digest_len = HEX_DIGEST_LEN + 1;
+    sep = "+";
+    b64_256 = 0;
+  }
+
+  if (lo < 0)
+    lo = 0;
+  if (hi > smartlist_len(digests))
+    hi = smartlist_len(digests);
+
+  if (hi-lo <= 0)
+    return;
+
+  tmp = smartlist_new();
+
+  for (; lo < hi; ++lo) {
+    cp = tor_malloc(enc_digest_len);
+    if (b64_256) {
+      digest256_to_base64(cp, smartlist_get(digests, lo));
+    } else {
+      base16_encode(cp, enc_digest_len, smartlist_get(digests, lo),
+                    digest_len);
+    }
+    smartlist_add(tmp, cp);
+  }
+
+  cp = smartlist_join_strings(tmp, sep, 0, NULL);
+  tor_asprintf(&resource, "d/%s.z", cp);
+
+  SMARTLIST_FOREACH(tmp, char *, cp1, tor_free(cp1));
+  smartlist_free(tmp);
+  tor_free(cp);
+
+  if (source) {
+    /* We know which authority or directory mirror we want. */
+    directory_request_t *req = directory_request_new(purpose);
+    directory_request_set_routerstatus(req, source);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
+  } else {
+    directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource,
+                                 pds_flags, DL_WANT_ANY_DIRSERVER);
+  }
+  tor_free(resource);
+}
+
+/** Return the max number of hashes to put in a URL for a given request.
+ */
+static int
+max_dl_per_request(const or_options_t *options, int purpose)
+{
+  /* Since squid does not like URLs >= 4096 bytes we limit it to 96.
+   *   4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535
+   *                 /tor/server/d/.z) == 4026
+   *   4026/41 (40 for the hash and 1 for the + that separates them) => 98
+   *   So use 96 because it's a nice number.
+   *
+   * For microdescriptors, the calculation is
+   *   4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535
+   *                 /tor/micro/d/.z) == 4027
+   *   4027/44 (43 for the hash and 1 for the - that separates them) => 91
+   *   So use 90 because it's a nice number.
+   */
+  int max = 96;
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    max = 90;
+  }
+  /* If we're going to tunnel our connections, we can ask for a lot more
+   * in a request. */
+  if (directory_must_use_begindir(options)) {
+    max = 500;
+  }
+  return max;
+}
+
+/** Don't split our requests so finely that we are requesting fewer than
+ * this number per server. (Grouping more than this at once leads to
+ * diminishing returns.) */
+#define MIN_DL_PER_REQUEST 32
+/** To prevent a single screwy cache from confusing us by selective reply,
+ * try to split our requests into at least this many requests. */
+#define MIN_REQUESTS 3
+/** If we want fewer than this many descriptors, wait until we
+ * want more, or until TestingClientMaxIntervalWithoutRequest has passed. */
+#define MAX_DL_TO_DELAY 16
+
+/** Given a <b>purpose</b> (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of
+ * router descriptor digests or microdescriptor digest256s in
+ * <b>downloadable</b>, decide whether to delay fetching until we have more.
+ * If we don't want to delay, launch one or more requests to the appropriate
+ * directory authorities.
+ */
+void
+launch_descriptor_downloads(int purpose,
+                            smartlist_t *downloadable,
+                            const routerstatus_t *source, time_t now)
+{
+  const or_options_t *options = get_options();
+  const char *descname;
+  const int fetch_microdesc = (purpose == DIR_PURPOSE_FETCH_MICRODESC);
+  int n_downloadable = smartlist_len(downloadable);
+
+  int i, n_per_request, max_dl_per_req;
+  const char *req_plural = "", *rtr_plural = "";
+  int pds_flags = PDS_RETRY_IF_NO_SERVERS;
+
+  tor_assert(fetch_microdesc || purpose == DIR_PURPOSE_FETCH_SERVERDESC);
+  descname = fetch_microdesc ? "microdesc" : "routerdesc";
+
+  if (!n_downloadable)
+    return;
+
+  if (!directory_fetches_dir_info_early(options)) {
+    if (n_downloadable >= MAX_DL_TO_DELAY) {
+      log_debug(LD_DIR,
+                "There are enough downloadable %ss to launch requests.",
+                descname);
+    } else if (! router_have_minimum_dir_info()) {
+      log_debug(LD_DIR,
+                "We are only missing %d %ss, but we'll fetch anyway, since "
+                "we don't yet have enough directory info.",
+                n_downloadable, descname);
+    } else {
+
+      /* should delay */
+      if ((last_descriptor_download_attempted +
+          options->TestingClientMaxIntervalWithoutRequest) > now)
+        return;
+
+      if (last_descriptor_download_attempted) {
+        log_info(LD_DIR,
+                 "There are not many downloadable %ss, but we've "
+                 "been waiting long enough (%d seconds). Downloading.",
+                 descname,
+                 (int)(now-last_descriptor_download_attempted));
+      } else {
+        log_info(LD_DIR,
+                 "There are not many downloadable %ss, but we haven't "
+                 "tried downloading descriptors recently. Downloading.",
+                 descname);
+      }
+    }
+  }
+
+  if (!authdir_mode(options)) {
+    /* If we wind up going to the authorities, we want to only open one
+     * connection to each authority at a time, so that we don't overload
+     * them.  We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH
+     * regardless of whether we're a cache or not.
+     *
+     * Setting this flag can make initiate_descriptor_downloads() ignore
+     * requests.  We need to make sure that we do in fact call
+     * update_router_descriptor_downloads() later on, once the connections
+     * have succeeded or failed.
+     */
+    pds_flags |= fetch_microdesc ?
+      PDS_NO_EXISTING_MICRODESC_FETCH :
+      PDS_NO_EXISTING_SERVERDESC_FETCH;
+  }
+
+  n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS);
+  max_dl_per_req = max_dl_per_request(options, purpose);
+
+  if (n_per_request > max_dl_per_req)
+    n_per_request = max_dl_per_req;
+
+  if (n_per_request < MIN_DL_PER_REQUEST) {
+    n_per_request = MIN(MIN_DL_PER_REQUEST, n_downloadable);
+  }
+
+  if (n_downloadable > n_per_request)
+    req_plural = rtr_plural = "s";
+  else if (n_downloadable > 1)
+    rtr_plural = "s";
+
+  log_info(LD_DIR,
+           "Launching %d request%s for %d %s%s, %d at a time",
+           CEIL_DIV(n_downloadable, n_per_request), req_plural,
+           n_downloadable, descname, rtr_plural, n_per_request);
+  smartlist_sort_digests(downloadable);
+  for (i=0; i < n_downloadable; i += n_per_request) {
+    initiate_descriptor_downloads(source, purpose,
+                                  downloadable, i, i+n_per_request,
+                                  pds_flags);
+  }
+  last_descriptor_download_attempted = now;
+}
+
+/** For any descriptor that we want that's currently listed in
+ * <b>consensus</b>, download it as appropriate. */
+void
+update_consensus_router_descriptor_downloads(time_t now, int is_vote,
+                                             networkstatus_t *consensus)
+{
+  const or_options_t *options = get_options();
+  digestmap_t *map = NULL;
+  smartlist_t *no_longer_old = smartlist_new();
+  smartlist_t *downloadable = smartlist_new();
+  routerstatus_t *source = NULL;
+  int authdir = authdir_mode(options);
+  int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0,
+    n_inprogress=0, n_in_oldrouters=0;
+
+  if (directory_too_idle_to_fetch_descriptors(options, now))
+    goto done;
+  if (!consensus)
+    goto done;
+
+  if (is_vote) {
+    /* where's it from, so we know whom to ask for descriptors */
+    dir_server_t *ds;
+    networkstatus_voter_info_t *voter = smartlist_get(consensus->voters, 0);
+    tor_assert(voter);
+    ds = trusteddirserver_get_by_v3_auth_digest(voter->identity_digest);
+    if (ds)
+      source = &(ds->fake_status);
+    else
+      log_warn(LD_DIR, "couldn't lookup source from vote?");
+  }
+
+  map = digestmap_new();
+  list_pending_descriptor_downloads(map, 0);
+  SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, void *, rsp) {
+      routerstatus_t *rs =
+        is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp;
+      signed_descriptor_t *sd;
+      if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) {
+        const routerinfo_t *ri;
+        ++n_have;
+        if (!(ri = router_get_by_id_digest(rs->identity_digest)) ||
+            tor_memneq(ri->cache_info.signed_descriptor_digest,
+                   sd->signed_descriptor_digest, DIGEST_LEN)) {
+          /* We have a descriptor with this digest, but either there is no
+           * entry in routerlist with the same ID (!ri), or there is one,
+           * but the identity digest differs (memneq).
+           */
+          smartlist_add(no_longer_old, sd);
+          ++n_in_oldrouters; /* We have it in old_routers. */
+        }
+        continue; /* We have it already. */
+      }
+      if (digestmap_get(map, rs->descriptor_digest)) {
+        ++n_inprogress;
+        continue; /* We have an in-progress download. */
+      }
+      if (!download_status_is_ready(&rs->dl_status, now)) {
+        ++n_delayed; /* Not ready for retry. */
+        continue;
+      }
+      if (authdir && dirserv_would_reject_router(rs)) {
+        ++n_would_reject;
+        continue; /* We would throw it out immediately. */
+      }
+      if (!we_want_to_fetch_flavor(options, consensus->flavor) &&
+          !client_would_use_router(rs, now)) {
+        ++n_wouldnt_use;
+        continue; /* We would never use it ourself. */
+      }
+      if (is_vote && source) {
+        char time_bufnew[ISO_TIME_LEN+1];
+        char time_bufold[ISO_TIME_LEN+1];
+        const routerinfo_t *oldrouter;
+        oldrouter = router_get_by_id_digest(rs->identity_digest);
+        format_iso_time(time_bufnew, rs->published_on);
+        if (oldrouter)
+          format_iso_time(time_bufold, oldrouter->cache_info.published_on);
+        log_info(LD_DIR, "Learned about %s (%s vs %s) from %s's vote (%s)",
+                 routerstatus_describe(rs),
+                 time_bufnew,
+                 oldrouter ? time_bufold : "none",
+                 source->nickname, oldrouter ? "known" : "unknown");
+      }
+      smartlist_add(downloadable, rs->descriptor_digest);
+  } SMARTLIST_FOREACH_END(rsp);
+
+  if (!authdir_mode_v3(options)
+      && smartlist_len(no_longer_old)) {
+    routerlist_t *rl = router_get_routerlist();
+    log_info(LD_DIR, "%d router descriptors listed in consensus are "
+             "currently in old_routers; making them current.",
+             smartlist_len(no_longer_old));
+    SMARTLIST_FOREACH_BEGIN(no_longer_old, signed_descriptor_t *, sd) {
+        const char *msg;
+        was_router_added_t r;
+        time_t tmp_cert_expiration_time;
+        routerinfo_t *ri = routerlist_reparse_old(rl, sd);
+        if (!ri) {
+          log_warn(LD_BUG, "Failed to re-parse a router.");
+          continue;
+        }
+        /* need to remember for below, since add_to_routerlist may free. */
+        tmp_cert_expiration_time = ri->cert_expiration_time;
+
+        r = router_add_to_routerlist(ri, &msg, 1, 0);
+        if (WRA_WAS_OUTDATED(r)) {
+          log_warn(LD_DIR, "Couldn't add re-parsed router: %s. This isn't "
+                   "usually a big deal, but you should make sure that your "
+                   "clock and timezone are set correctly.",
+                   msg?msg:"???");
+          if (r == ROUTER_CERTS_EXPIRED) {
+            char time_cons[ISO_TIME_LEN+1];
+            char time_cert_expires[ISO_TIME_LEN+1];
+            format_iso_time(time_cons, consensus->valid_after);
+            format_iso_time(time_cert_expires, tmp_cert_expiration_time);
+            log_warn(LD_DIR, "  (I'm looking at a consensus from %s; This "
+                     "router's certificates began expiring at %s.)",
+                     time_cons, time_cert_expires);
+          }
+        }
+    } SMARTLIST_FOREACH_END(sd);
+    routerlist_assert_ok(rl);
+  }
+
+  log_info(LD_DIR,
+           "%d router descriptors downloadable. %d delayed; %d present "
+           "(%d of those were in old_routers); %d would_reject; "
+           "%d wouldnt_use; %d in progress.",
+           smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters,
+           n_would_reject, n_wouldnt_use, n_inprogress);
+
+  launch_descriptor_downloads(DIR_PURPOSE_FETCH_SERVERDESC,
+                              downloadable, source, now);
+
+  digestmap_free(map, NULL);
+ done:
+  smartlist_free(downloadable);
+  smartlist_free(no_longer_old);
+}
+
+/** How often should we launch a server/authority request to be sure of getting
+ * a guess for our IP? */
+/*XXXX+ this info should come from netinfo cells or something, or we should
+ * do this only when we aren't seeing incoming data. see bug 652. */
+#define DUMMY_DOWNLOAD_INTERVAL (20*60)
+
+/** As needed, launch a dummy router descriptor fetch to see if our
+ * address has changed. */
+static void
+launch_dummy_descriptor_download_as_needed(time_t now,
+                                           const or_options_t *options)
+{
+  static time_t last_dummy_download = 0;
+  /* XXXX+ we could be smarter here; see notes on bug 652. */
+  /* If we're a server that doesn't have a configured address, we rely on
+   * directory fetches to learn when our address changes.  So if we haven't
+   * tried to get any routerdescs in a long time, try a dummy fetch now. */
+  if (!options->Address &&
+      server_mode(options) &&
+      last_descriptor_download_attempted + DUMMY_DOWNLOAD_INTERVAL < now &&
+      last_dummy_download + DUMMY_DOWNLOAD_INTERVAL < now) {
+    last_dummy_download = now;
+    /* XX/teor - do we want an authority here, because they are less likely
+     * to give us the wrong address? (See #17782)
+     * I'm leaving the previous behaviour intact, because I don't like
+     * the idea of some relays contacting an authority every 20 minutes. */
+    directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
+                                 ROUTER_PURPOSE_GENERAL, "authority.z",
+                                 PDS_RETRY_IF_NO_SERVERS,
+                                 DL_WANT_ANY_DIRSERVER);
+  }
+}
+
+/** Launch downloads for router status as needed. */
+void
+update_router_descriptor_downloads(time_t now)
+{
+  const or_options_t *options = get_options();
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+  if (!we_fetch_router_descriptors(options))
+    return;
+
+  update_consensus_router_descriptor_downloads(now, 0,
+                  networkstatus_get_reasonably_live_consensus(now, FLAV_NS));
+}
+
+/** Launch extrainfo downloads as needed. */
+void
+update_extrainfo_downloads(time_t now)
+{
+  const or_options_t *options = get_options();
+  routerlist_t *rl;
+  smartlist_t *wanted;
+  digestmap_t *pending;
+  int old_routers, i, max_dl_per_req;
+  int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0, n_bogus[2] = {0,0};
+  if (! options->DownloadExtraInfo)
+    return;
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+  if (!router_have_minimum_dir_info())
+    return;
+
+  pending = digestmap_new();
+  list_pending_descriptor_downloads(pending, 1);
+  rl = router_get_routerlist();
+  wanted = smartlist_new();
+  for (old_routers = 0; old_routers < 2; ++old_routers) {
+    smartlist_t *lst = old_routers ? rl->old_routers : rl->routers;
+    for (i = 0; i < smartlist_len(lst); ++i) {
+      signed_descriptor_t *sd;
+      char *d;
+      if (old_routers)
+        sd = smartlist_get(lst, i);
+      else
+        sd = &((routerinfo_t*)smartlist_get(lst, i))->cache_info;
+      if (sd->is_extrainfo)
+        continue; /* This should never happen. */
+      if (old_routers && !router_get_by_id_digest(sd->identity_digest))
+        continue; /* Couldn't check the signature if we got it. */
+      if (sd->extrainfo_is_bogus)
+        continue;
+      d = sd->extra_info_digest;
+      if (tor_digest_is_zero(d)) {
+        ++n_no_ei;
+        continue;
+      }
+      if (eimap_get(rl->extra_info_map, d)) {
+        ++n_have;
+        continue;
+      }
+      if (!download_status_is_ready(&sd->ei_dl_status, now)) {
+        ++n_delay;
+        continue;
+      }
+      if (digestmap_get(pending, d)) {
+        ++n_pending;
+        continue;
+      }
+
+      const signed_descriptor_t *sd2 = router_get_by_extrainfo_digest(d);
+      if (sd2 != sd) {
+        if (sd2 != NULL) {
+          char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1];
+          char d3[HEX_DIGEST_LEN+1], d4[HEX_DIGEST_LEN+1];
+          base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN);
+          base16_encode(d2, sizeof(d2), sd2->identity_digest, DIGEST_LEN);
+          base16_encode(d3, sizeof(d3), d, DIGEST_LEN);
+          base16_encode(d4, sizeof(d3), sd2->extra_info_digest, DIGEST_LEN);
+
+          log_info(LD_DIR, "Found an entry in %s with mismatched "
+                   "router_get_by_extrainfo_digest() value. This has ID %s "
+                   "but the entry in the map has ID %s. This has EI digest "
+                   "%s and the entry in the map has EI digest %s.",
+                   old_routers?"old_routers":"routers",
+                   d1, d2, d3, d4);
+        } else {
+          char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1];
+          base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN);
+          base16_encode(d2, sizeof(d2), d, DIGEST_LEN);
+
+          log_info(LD_DIR, "Found an entry in %s with NULL "
+                   "router_get_by_extrainfo_digest() value. This has ID %s "
+                   "and EI digest %s.",
+                   old_routers?"old_routers":"routers",
+                   d1, d2);
+        }
+        ++n_bogus[old_routers];
+        continue;
+      }
+      smartlist_add(wanted, d);
+    }
+  }
+  digestmap_free(pending, NULL);
+
+  log_info(LD_DIR, "Extrainfo download status: %d router with no ei, %d "
+           "with present ei, %d delaying, %d pending, %d downloadable, %d "
+           "bogus in routers, %d bogus in old_routers",
+           n_no_ei, n_have, n_delay, n_pending, smartlist_len(wanted),
+           n_bogus[0], n_bogus[1]);
+
+  smartlist_shuffle(wanted);
+
+  max_dl_per_req = max_dl_per_request(options, DIR_PURPOSE_FETCH_EXTRAINFO);
+  for (i = 0; i < smartlist_len(wanted); i += max_dl_per_req) {
+    initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_EXTRAINFO,
+                                  wanted, i, i+max_dl_per_req,
+                PDS_RETRY_IF_NO_SERVERS|PDS_NO_EXISTING_SERVERDESC_FETCH);
+  }
+
+  smartlist_free(wanted);
+}
+
+/** Reset the consensus and extra-info download failure count on all routers.
+ * When we get a new consensus,
+ * routers_update_status_from_consensus_networkstatus() will reset the
+ * download statuses on the descriptors in that consensus.
+ */
+void
+router_reset_descriptor_download_failures(void)
+{
+  log_debug(LD_GENERAL,
+            "In router_reset_descriptor_download_failures()");
+
+  networkstatus_reset_download_failures();
+  last_descriptor_download_attempted = 0;
+  if (!routerlist)
+    return;
+  /* We want to download *all* extra-info descriptors, not just those in
+   * the consensus we currently have (or are about to have) */
+  SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri,
+  {
+    download_status_reset(&ri->cache_info.ei_dl_status);
+  });
+  SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd,
+  {
+    download_status_reset(&sd->ei_dl_status);
+  });
+}
+
+/** Any changes in a router descriptor's publication time larger than this are
+ * automatically non-cosmetic. */
+#define ROUTER_MAX_COSMETIC_TIME_DIFFERENCE (2*60*60)
+
+/** We allow uptime to vary from how much it ought to be by this much. */
+#define ROUTER_ALLOW_UPTIME_DRIFT (6*60*60)
+
+/** Return true iff the only differences between r1 and r2 are such that
+ * would not cause a recent (post 0.1.1.6) dirserver to republish.
+ */
+int
+router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2)
+{
+  time_t r1pub, r2pub;
+  long time_difference;
+  tor_assert(r1 && r2);
+
+  /* r1 should be the one that was published first. */
+  if (r1->cache_info.published_on > r2->cache_info.published_on) {
+    const routerinfo_t *ri_tmp = r2;
+    r2 = r1;
+    r1 = ri_tmp;
+  }
+
+  /* If any key fields differ, they're different. */
+  if (r1->addr != r2->addr ||
+      strcasecmp(r1->nickname, r2->nickname) ||
+      r1->or_port != r2->or_port ||
+      !tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) ||
+      r1->ipv6_orport != r2->ipv6_orport ||
+      r1->dir_port != r2->dir_port ||
+      r1->purpose != r2->purpose ||
+      !crypto_pk_eq_keys(r1->onion_pkey, r2->onion_pkey) ||
+      !crypto_pk_eq_keys(r1->identity_pkey, r2->identity_pkey) ||
+      strcasecmp(r1->platform, r2->platform) ||
+      (r1->contact_info && !r2->contact_info) || /* contact_info is optional */
+      (!r1->contact_info && r2->contact_info) ||
+      (r1->contact_info && r2->contact_info &&
+       strcasecmp(r1->contact_info, r2->contact_info)) ||
+      r1->is_hibernating != r2->is_hibernating ||
+      ! addr_policies_eq(r1->exit_policy, r2->exit_policy) ||
+      (r1->supports_tunnelled_dir_requests !=
+       r2->supports_tunnelled_dir_requests))
+    return 0;
+  if ((r1->declared_family == NULL) != (r2->declared_family == NULL))
+    return 0;
+  if (r1->declared_family && r2->declared_family) {
+    int i, n;
+    if (smartlist_len(r1->declared_family)!=smartlist_len(r2->declared_family))
+      return 0;
+    n = smartlist_len(r1->declared_family);
+    for (i=0; i < n; ++i) {
+      if (strcasecmp(smartlist_get(r1->declared_family, i),
+                     smartlist_get(r2->declared_family, i)))
+        return 0;
+    }
+  }
+
+  /* Did bandwidth change a lot? */
+  if ((r1->bandwidthcapacity < r2->bandwidthcapacity/2) ||
+      (r2->bandwidthcapacity < r1->bandwidthcapacity/2))
+    return 0;
+
+  /* Did the bandwidthrate or bandwidthburst change? */
+  if ((r1->bandwidthrate != r2->bandwidthrate) ||
+      (r1->bandwidthburst != r2->bandwidthburst))
+    return 0;
+
+  /* Did more than 12 hours pass? */
+  if (r1->cache_info.published_on + ROUTER_MAX_COSMETIC_TIME_DIFFERENCE
+      < r2->cache_info.published_on)
+    return 0;
+
+  /* Did uptime fail to increase by approximately the amount we would think,
+   * give or take some slop? */
+  r1pub = r1->cache_info.published_on;
+  r2pub = r2->cache_info.published_on;
+  time_difference = labs(r2->uptime - (r1->uptime + (r2pub - r1pub)));
+  if (time_difference > ROUTER_ALLOW_UPTIME_DRIFT &&
+      time_difference > r1->uptime * .05 &&
+      time_difference > r2->uptime * .05)
+    return 0;
+
+  /* Otherwise, the difference is cosmetic. */
+  return 1;
+}
+
+/** Check whether <b>sd</b> describes a router descriptor compatible with the
+ * extrainfo document <b>ei</b>.
+ *
+ * <b>identity_pkey</b> (which must also be provided) is RSA1024 identity key
+ * for the router. We use it to check the signature of the extrainfo document,
+ * if it has not already been checked.
+ *
+ * If no router is compatible with <b>ei</b>, <b>ei</b> should be
+ * dropped.  Return 0 for "compatible", return 1 for "reject, and inform
+ * whoever uploaded <b>ei</b>, and return -1 for "reject silently.".  If
+ * <b>msg</b> is present, set *<b>msg</b> to a description of the
+ * incompatibility (if any).
+ *
+ * Set the extrainfo_is_bogus field in <b>sd</b> if the digests matched
+ * but the extrainfo was nonetheless incompatible.
+ **/
+int
+routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey,
+                                       extrainfo_t *ei,
+                                       signed_descriptor_t *sd,
+                                       const char **msg)
+{
+  int digest_matches, digest256_matches, r=1;
+  tor_assert(identity_pkey);
+  tor_assert(sd);
+  tor_assert(ei);
+
+  if (ei->bad_sig) {
+    if (msg) *msg = "Extrainfo signature was bad, or signed with wrong key.";
+    return 1;
+  }
+
+  digest_matches = tor_memeq(ei->cache_info.signed_descriptor_digest,
+                           sd->extra_info_digest, DIGEST_LEN);
+  /* Set digest256_matches to 1 if the digest is correct, or if no
+   * digest256 was in the ri. */
+  digest256_matches = tor_memeq(ei->digest256,
+                                sd->extra_info_digest256, DIGEST256_LEN);
+  digest256_matches |=
+    tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
+
+  /* The identity must match exactly to have been generated at the same time
+   * by the same router. */
+  if (tor_memneq(sd->identity_digest,
+                 ei->cache_info.identity_digest,
+                 DIGEST_LEN)) {
+    if (msg) *msg = "Extrainfo nickname or identity did not match routerinfo";
+    goto err; /* different servers */
+  }
+
+  if (! tor_cert_opt_eq(sd->signing_key_cert,
+                        ei->cache_info.signing_key_cert)) {
+    if (msg) *msg = "Extrainfo signing key cert didn't match routerinfo";
+    goto err; /* different servers */
+  }
+
+  if (ei->pending_sig) {
+    char signed_digest[128];
+    if (crypto_pk_public_checksig(identity_pkey,
+                       signed_digest, sizeof(signed_digest),
+                       ei->pending_sig, ei->pending_sig_len) != DIGEST_LEN ||
+        tor_memneq(signed_digest, ei->cache_info.signed_descriptor_digest,
+               DIGEST_LEN)) {
+      ei->bad_sig = 1;
+      tor_free(ei->pending_sig);
+      if (msg) *msg = "Extrainfo signature bad, or signed with wrong key";
+      goto err; /* Bad signature, or no match. */
+    }
+
+    ei->cache_info.send_unencrypted = sd->send_unencrypted;
+    tor_free(ei->pending_sig);
+  }
+
+  if (ei->cache_info.published_on < sd->published_on) {
+    if (msg) *msg = "Extrainfo published time did not match routerdesc";
+    goto err;
+  } else if (ei->cache_info.published_on > sd->published_on) {
+    if (msg) *msg = "Extrainfo published time did not match routerdesc";
+    r = -1;
+    goto err;
+  }
+
+  if (!digest256_matches && !digest_matches) {
+    if (msg) *msg = "Neither digest256 or digest matched "
+               "digest from routerdesc";
+    goto err;
+  }
+
+  if (!digest256_matches) {
+    if (msg) *msg = "Extrainfo digest did not match digest256 from routerdesc";
+    goto err; /* Digest doesn't match declared value. */
+  }
+
+  if (!digest_matches) {
+    if (msg) *msg = "Extrainfo digest did not match value from routerdesc";
+    goto err; /* Digest doesn't match declared value. */
+  }
+
+  return 0;
+ err:
+  if (digest_matches) {
+    /* This signature was okay, and the digest was right: This is indeed the
+     * corresponding extrainfo.  But insanely, it doesn't match the routerinfo
+     * that lists it.  Don't try to fetch this one again. */
+    sd->extrainfo_is_bogus = 1;
+  }
+
+  return r;
+}
+
+/* Does ri have a valid ntor onion key?
+ * Valid ntor onion keys exist and have at least one non-zero byte. */
+int
+routerinfo_has_curve25519_onion_key(const routerinfo_t *ri)
+{
+  if (!ri) {
+    return 0;
+  }
+
+  if (!ri->onion_curve25519_pkey) {
+    return 0;
+  }
+
+  if (tor_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
+                      CURVE25519_PUBKEY_LEN)) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/* Is rs running a tor version known to support EXTEND2 cells?
+ * If allow_unknown_versions is true, return true if we can't tell
+ * (from a versions line or a protocols line) whether it supports extend2
+ * cells.
+ * Otherwise, return false if the version is unknown. */
+int
+routerstatus_version_supports_extend2_cells(const routerstatus_t *rs,
+                                            int allow_unknown_versions)
+{
+  if (!rs) {
+    return allow_unknown_versions;
+  }
+
+  if (!rs->pv.protocols_known) {
+    return allow_unknown_versions;
+  }
+
+  return rs->pv.supports_extend2_cells;
+}
+
+/** Assert that the internal representation of <b>rl</b> is
+ * self-consistent. */
+void
+routerlist_assert_ok(const routerlist_t *rl)
+{
+  routerinfo_t *r2;
+  signed_descriptor_t *sd2;
+  if (!rl)
+    return;
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, r) {
+    r2 = rimap_get(rl->identity_map, r->cache_info.identity_digest);
+    tor_assert(r == r2);
+    sd2 = sdmap_get(rl->desc_digest_map,
+                    r->cache_info.signed_descriptor_digest);
+    tor_assert(&(r->cache_info) == sd2);
+    tor_assert(r->cache_info.routerlist_index == r_sl_idx);
+    /* XXXX
+     *
+     *   Hoo boy.  We need to fix this one, and the fix is a bit tricky, so
+     * commenting this out is just a band-aid.
+     *
+     *   The problem is that, although well-behaved router descriptors
+     * should never have the same value for their extra_info_digest, it's
+     * possible for ill-behaved routers to claim whatever they like there.
+     *
+     *   The real answer is to trash desc_by_eid_map and instead have
+     * something that indicates for a given extra-info digest we want,
+     * what its download status is.  We'll do that as a part of routerlist
+     * refactoring once consensus directories are in.  For now,
+     * this rep violation is probably harmless: an adversary can make us
+     * reset our retry count for an extrainfo, but that's not the end
+     * of the world.  Changing the representation in 0.2.0.x would just
+     * destabilize the codebase.
+    if (!tor_digest_is_zero(r->cache_info.extra_info_digest)) {
+      signed_descriptor_t *sd3 =
+        sdmap_get(rl->desc_by_eid_map, r->cache_info.extra_info_digest);
+      tor_assert(sd3 == &(r->cache_info));
+    }
+    */
+  } SMARTLIST_FOREACH_END(r);
+  SMARTLIST_FOREACH_BEGIN(rl->old_routers, signed_descriptor_t *, sd) {
+    r2 = rimap_get(rl->identity_map, sd->identity_digest);
+    tor_assert(!r2 || sd != &(r2->cache_info));
+    sd2 = sdmap_get(rl->desc_digest_map, sd->signed_descriptor_digest);
+    tor_assert(sd == sd2);
+    tor_assert(sd->routerlist_index == sd_sl_idx);
+    /* XXXX see above.
+    if (!tor_digest_is_zero(sd->extra_info_digest)) {
+      signed_descriptor_t *sd3 =
+        sdmap_get(rl->desc_by_eid_map, sd->extra_info_digest);
+      tor_assert(sd3 == sd);
+    }
+    */
+  } SMARTLIST_FOREACH_END(sd);
+
+  RIMAP_FOREACH(rl->identity_map, d, r) {
+    tor_assert(tor_memeq(r->cache_info.identity_digest, d, DIGEST_LEN));
+  } DIGESTMAP_FOREACH_END;
+  SDMAP_FOREACH(rl->desc_digest_map, d, sd) {
+    tor_assert(tor_memeq(sd->signed_descriptor_digest, d, DIGEST_LEN));
+  } DIGESTMAP_FOREACH_END;
+  SDMAP_FOREACH(rl->desc_by_eid_map, d, sd) {
+    tor_assert(!tor_digest_is_zero(d));
+    tor_assert(sd);
+    tor_assert(tor_memeq(sd->extra_info_digest, d, DIGEST_LEN));
+  } DIGESTMAP_FOREACH_END;
+  EIMAP_FOREACH(rl->extra_info_map, d, ei) {
+    signed_descriptor_t *sd;
+    tor_assert(tor_memeq(ei->cache_info.signed_descriptor_digest,
+                       d, DIGEST_LEN));
+    sd = sdmap_get(rl->desc_by_eid_map,
+                   ei->cache_info.signed_descriptor_digest);
+    // tor_assert(sd); // XXXX see above
+    if (sd) {
+      tor_assert(tor_memeq(ei->cache_info.signed_descriptor_digest,
+                         sd->extra_info_digest, DIGEST_LEN));
+    }
+  } DIGESTMAP_FOREACH_END;
+}
+
+/** Allocate and return a new string representing the contact info
+ * and platform string for <b>router</b>,
+ * surrounded by quotes and using standard C escapes.
+ *
+ * THIS FUNCTION IS NOT REENTRANT.  Don't call it from outside the main
+ * thread.  Also, each call invalidates the last-returned value, so don't
+ * try log_warn(LD_GENERAL, "%s %s", esc_router_info(a), esc_router_info(b));
+ *
+ * If <b>router</b> is NULL, it just frees its internal memory and returns.
+ */
+const char *
+esc_router_info(const routerinfo_t *router)
+{
+  static char *info=NULL;
+  char *esc_contact, *esc_platform;
+  tor_free(info);
+
+  if (!router)
+    return NULL; /* we're exiting; just free the memory we use */
+
+  esc_contact = esc_for_log(router->contact_info);
+  esc_platform = esc_for_log(router->platform);
+
+  tor_asprintf(&info, "Contact %s, Platform %s", esc_contact, esc_platform);
+  tor_free(esc_contact);
+  tor_free(esc_platform);
+
+  return info;
+}
+
+/** Helper for sorting: compare two routerinfos by their identity
+ * digest. */
+static int
+compare_routerinfo_by_id_digest_(const void **a, const void **b)
+{
+  routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b;
+  return fast_memcmp(first->cache_info.identity_digest,
+                second->cache_info.identity_digest,
+                DIGEST_LEN);
+}
+
+/** Sort a list of routerinfo_t in ascending order of identity digest. */
+void
+routers_sort_by_identity(smartlist_t *routers)
+{
+  smartlist_sort(routers, compare_routerinfo_by_id_digest_);
+}
+
+/** Called when we change a node set, or when we reload the geoip IPv4 list:
+ * recompute all country info in all configuration node sets and in the
+ * routerlist. */
+void
+refresh_all_country_info(void)
+{
+  const or_options_t *options = get_options();
+
+  if (options->EntryNodes)
+    routerset_refresh_countries(options->EntryNodes);
+  if (options->ExitNodes)
+    routerset_refresh_countries(options->ExitNodes);
+  if (options->ExcludeNodes)
+    routerset_refresh_countries(options->ExcludeNodes);
+  if (options->ExcludeExitNodes)
+    routerset_refresh_countries(options->ExcludeExitNodes);
+  if (options->ExcludeExitNodesUnion_)
+    routerset_refresh_countries(options->ExcludeExitNodesUnion_);
+
+  nodelist_refresh_countries();
+}
diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h
new file mode 100644
index 0000000000..4b7406364f
--- /dev/null
+++ b/src/feature/nodelist/routerlist.h
@@ -0,0 +1,347 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerlist.h
+ * \brief Header file for routerlist.c.
+ **/
+
+#ifndef TOR_ROUTERLIST_H
+#define TOR_ROUTERLIST_H
+
+#include "lib/testsupport/testsupport.h"
+
+/** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */
+typedef enum was_router_added_t {
+  /* Router was added successfully. */
+  ROUTER_ADDED_SUCCESSFULLY = 1,
+  /* Extrainfo document was rejected because no corresponding router
+   * descriptor was found OR router descriptor was rejected because
+   * it was incompatible with its extrainfo document. */
+  ROUTER_BAD_EI = -1,
+  /* Router descriptor was rejected because it is already known. */
+  ROUTER_IS_ALREADY_KNOWN = -2,
+  /* General purpose router was rejected, because it was not listed
+   * in consensus. */
+  ROUTER_NOT_IN_CONSENSUS = -3,
+  /* Router was neither in directory consensus nor in any of
+   * networkstatus documents. Caching it to access later.
+   * (Applies to fetched descriptors only.) */
+  ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS = -4,
+  /* Router was rejected by directory authority. */
+  ROUTER_AUTHDIR_REJECTS = -5,
+  /* Bridge descriptor was rejected because such bridge was not one
+   * of the bridges we have listed in our configuration. */
+  ROUTER_WAS_NOT_WANTED = -6,
+  /* Router descriptor was rejected because it was older than
+   * OLD_ROUTER_DESC_MAX_AGE. */
+  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */
+  /* DOCDOC */
+  ROUTER_CERTS_EXPIRED = -8
+} was_router_added_t;
+
+/** Flags to be passed to control router_choose_random_node() to indicate what
+ * kind of nodes to pick according to what algorithm. */
+typedef enum router_crn_flags_t {
+  CRN_NEED_UPTIME = 1<<0,
+  CRN_NEED_CAPACITY = 1<<1,
+  CRN_NEED_GUARD = 1<<2,
+  /* XXXX not used, apparently. */
+  CRN_WEIGHT_AS_EXIT = 1<<5,
+  CRN_NEED_DESC = 1<<6,
+  /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */
+  CRN_PREF_ADDR = 1<<7,
+  /* On clients, only provide nodes that we can connect to directly, based on
+   * our firewall rules */
+  CRN_DIRECT_CONN = 1<<8,
+  /* On clients, only provide nodes with HSRend >= 2 protocol version which
+   * is required for hidden service version >= 3. */
+  CRN_RENDEZVOUS_V3 = 1<<9,
+} router_crn_flags_t;
+
+/** Possible ways to weight routers when choosing one randomly.  See
+ * routerlist_sl_choose_by_bandwidth() for more information.*/
+typedef enum bandwidth_weight_rule_t {
+  NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD,
+  WEIGHT_FOR_DIR
+} bandwidth_weight_rule_t;
+
+/* Flags for pick_directory_server() and pick_trusteddirserver(). */
+/** Flag to indicate that we should not automatically be willing to use
+ * ourself to answer a directory request.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_ALLOW_SELF                 (1<<0)
+/** Flag to indicate that if no servers seem to be up, we should mark all
+ * directory servers as up and try again.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_RETRY_IF_NO_SERVERS        (1<<1)
+/** Flag to indicate that we should not exclude directory servers that
+ * our ReachableAddress settings would exclude.  This usually means that
+ * we're going to connect to the server over Tor, and so we don't need to
+ * worry about our firewall telling us we can't.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_IGNORE_FASCISTFIREWALL     (1<<2)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading server descriptors
+ * or extrainfo documents.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading microdescs.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
+
+int get_n_authorities(dirinfo_type_t type);
+int trusted_dirs_reload_certs(void);
+
+/*
+ * Pass one of these as source to trusted_dirs_load_certs_from_string()
+ * to indicate whence string originates; this controls error handling
+ * behavior such as marking downloads as failed.
+ */
+
+#define TRUSTED_DIRS_CERTS_SRC_SELF 0
+#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3
+#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4
+
+int trusted_dirs_load_certs_from_string(const char *contents, int source,
+                                        int flush, const char *source_dir);
+void trusted_dirs_flush_certs_to_disk(void);
+authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest);
+authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest);
+authority_cert_t *authority_cert_get_by_digests(const char *id_digest,
+                                                const char *sk_digest);
+void authority_cert_get_all(smartlist_t *certs_out);
+void authority_cert_dl_failed(const char *id_digest,
+                              const char *signing_key_digest, int status);
+void authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+                                   const char *dir_hint);
+int router_reload_router_list(void);
+int authority_cert_dl_looks_uncertain(const char *id_digest);
+const smartlist_t *router_get_trusted_dir_servers(void);
+const smartlist_t *router_get_fallback_dir_servers(void);
+int authority_cert_is_blacklisted(const authority_cert_t *cert);
+
+const routerstatus_t *router_pick_directory_server(dirinfo_type_t type,
+                                                   int flags);
+dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
+dir_server_t *router_get_fallback_dirserver_by_digest(
+                                                   const char *digest);
+int router_digest_is_fallback_dir(const char *digest);
+MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest,
+          (const char *d));
+const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type,
+                                                   int flags);
+const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type,
+                                                     int flags);
+int router_skip_or_reachability(const or_options_t *options, int try_ip_pref);
+int router_get_my_share_of_directory_requests(double *v3_share_out);
+void router_reset_status_download_failures(void);
+int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2);
+void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                           int need_capacity, int need_guard,
+                                           int need_desc, int pref_addr,
+                                           int direct_conn);
+
+const routerinfo_t *routerlist_find_my_routerinfo(void);
+uint32_t router_get_advertised_bandwidth(const routerinfo_t *router);
+uint32_t router_get_advertised_bandwidth_capped(const routerinfo_t *router);
+
+const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl,
+                                          bandwidth_weight_rule_t rule);
+double frac_nodes_with_descriptors(const smartlist_t *sl,
+                                   bandwidth_weight_rule_t rule,
+                                   int for_direct_conn);
+
+const node_t *router_choose_random_node(smartlist_t *excludedsmartlist,
+                                        struct routerset_t *excludedset,
+                                        router_crn_flags_t flags);
+
+int router_digest_is_trusted_dir_type(const char *digest,
+                                      dirinfo_type_t type);
+#define router_digest_is_trusted_dir(d) \
+  router_digest_is_trusted_dir_type((d), NO_DIRINFO)
+
+int hexdigest_to_digest(const char *hexdigest, char *digest);
+const routerinfo_t *router_get_by_id_digest(const char *digest);
+routerinfo_t *router_get_mutable_by_digest(const char *digest);
+signed_descriptor_t *router_get_by_descriptor_digest(const char *digest);
+MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest,
+          (const char *digest));
+MOCK_DECL(signed_descriptor_t *,extrainfo_get_by_descriptor_digest,
+          (const char *digest));
+const char *signed_descriptor_get_body(const signed_descriptor_t *desc);
+const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc);
+routerlist_t *router_get_routerlist(void);
+void routerinfo_free_(routerinfo_t *router);
+#define routerinfo_free(router) \
+  FREE_AND_NULL(routerinfo_t, routerinfo_free_, (router))
+void extrainfo_free_(extrainfo_t *extrainfo);
+#define extrainfo_free(ei) FREE_AND_NULL(extrainfo_t, extrainfo_free_, (ei))
+void routerlist_free_(routerlist_t *rl);
+#define routerlist_free(rl) FREE_AND_NULL(routerlist_t, routerlist_free_, (rl))
+void dump_routerlist_mem_usage(int severity);
+void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old,
+                       time_t now);
+void routerlist_free_all(void);
+void routerlist_reset_warnings(void);
+
+MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void));
+MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id,
+          (const char *digest));
+MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id,
+          (const char *digest));
+MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
+    (const char *id_digest, const char *sk_digest));
+
+static int WRA_WAS_ADDED(was_router_added_t s);
+static int WRA_WAS_OUTDATED(was_router_added_t s);
+static int WRA_WAS_REJECTED(was_router_added_t s);
+static int WRA_NEVER_DOWNLOADABLE(was_router_added_t s);
+/** Return true iff the outcome code in <b>s</b> indicates that the descriptor
+ * was added. It might still be necessary to check whether the descriptor
+ * generator should be notified.
+ */
+static inline int
+WRA_WAS_ADDED(was_router_added_t s) {
+  return s == ROUTER_ADDED_SUCCESSFULLY;
+}
+/** Return true iff the outcome code in <b>s</b> indicates that the descriptor
+ * was not added because it was either:
+ * - not in the consensus
+ * - neither in the consensus nor in any networkstatus document
+ * - it was outdated.
+ * - its certificates were expired.
+ */
+static inline int WRA_WAS_OUTDATED(was_router_added_t s)
+{
+  return (s == ROUTER_WAS_TOO_OLD ||
+          s == ROUTER_IS_ALREADY_KNOWN ||
+          s == ROUTER_NOT_IN_CONSENSUS ||
+          s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS ||
+          s == ROUTER_CERTS_EXPIRED);
+}
+/** Return true iff the outcome code in <b>s</b> indicates that the descriptor
+ * was flat-out rejected. */
+static inline int WRA_WAS_REJECTED(was_router_added_t s)
+{
+  return (s == ROUTER_AUTHDIR_REJECTS);
+}
+/** Return true iff the outcome code in <b>s</b> indicates that the descriptor
+ * was flat-out rejected. */
+static inline int WRA_NEVER_DOWNLOADABLE(was_router_added_t s)
+{
+  return (s == ROUTER_AUTHDIR_REJECTS ||
+          s == ROUTER_BAD_EI ||
+          s == ROUTER_WAS_TOO_OLD ||
+          s == ROUTER_CERTS_EXPIRED);
+}
+was_router_added_t router_add_to_routerlist(routerinfo_t *router,
+                                            const char **msg,
+                                            int from_cache,
+                                            int from_fetch);
+was_router_added_t router_add_extrainfo_to_routerlist(
+                                        extrainfo_t *ei, const char **msg,
+                                        int from_cache, int from_fetch);
+void routerlist_descriptors_added(smartlist_t *sl, int from_cache);
+void routerlist_remove_old_routers(void);
+int router_load_single_router(const char *s, uint8_t purpose, int cache,
+                              const char **msg);
+int router_load_routers_from_string(const char *s, const char *eos,
+                                     saved_location_t saved_location,
+                                     smartlist_t *requested_fingerprints,
+                                     int descriptor_digests,
+                                     const char *prepend_annotations);
+void router_load_extrainfo_from_string(const char *s, const char *eos,
+                                       saved_location_t saved_location,
+                                       smartlist_t *requested_fingerprints,
+                                       int descriptor_digests);
+
+void routerlist_retry_directory_downloads(time_t now);
+
+int router_exit_policy_rejects_all(const routerinfo_t *router);
+
+dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const tor_addr_port_t *addrport_ipv6,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight);
+dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
+                                      uint16_t dir_port, uint16_t or_port,
+                                      const tor_addr_port_t *addrport_ipv6,
+                                      const char *id_digest, double weight);
+void dir_server_add(dir_server_t *ent);
+
+void authority_cert_free_(authority_cert_t *cert);
+#define authority_cert_free(cert) \
+  FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert))
+void clear_dir_servers(void);
+void update_consensus_router_descriptor_downloads(time_t now, int is_vote,
+                                                  networkstatus_t *consensus);
+void update_router_descriptor_downloads(time_t now);
+void update_all_descriptor_downloads(time_t now);
+void update_extrainfo_downloads(time_t now);
+void router_reset_descriptor_download_failures(void);
+int router_differences_are_cosmetic(const routerinfo_t *r1,
+                                    const routerinfo_t *r2);
+int routerinfo_incompatible_with_extrainfo(const crypto_pk_t *ri,
+                                           extrainfo_t *ei,
+                                           signed_descriptor_t *sd,
+                                           const char **msg);
+int routerinfo_has_curve25519_onion_key(const routerinfo_t *ri);
+int routerstatus_version_supports_extend2_cells(const routerstatus_t *rs,
+                                                int allow_unknown_versions);
+
+void routerlist_assert_ok(const routerlist_t *rl);
+const char *esc_router_info(const routerinfo_t *router);
+void routers_sort_by_identity(smartlist_t *routers);
+
+void refresh_all_country_info(void);
+
+void list_pending_microdesc_downloads(digest256map_t *result);
+void launch_descriptor_downloads(int purpose,
+                                 smartlist_t *downloadable,
+                                 const routerstatus_t *source,
+                                 time_t now);
+
+int hex_digest_nickname_decode(const char *hexdigest,
+                               char *digest_out,
+                               char *nickname_qualifier_out,
+                               char *nickname_out);
+int hex_digest_nickname_matches(const char *hexdigest,
+                                const char *identity_digest,
+                                const char *nickname);
+
+#ifdef ROUTERLIST_PRIVATE
+STATIC int choose_array_element_by_weight(const uint64_t *entries,
+                                          int n_entries);
+STATIC void scale_array_elements_to_u64(uint64_t *entries_out,
+                                        const double *entries_in,
+                                        int n_entries,
+                                        uint64_t *total_out);
+STATIC const routerstatus_t *router_pick_directory_server_impl(
+                                           dirinfo_type_t auth, int flags,
+                                           int *n_busy_out);
+
+MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router,
+                                                 int seconds));
+MOCK_DECL(STATIC was_router_added_t, extrainfo_insert,
+          (routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible));
+
+MOCK_DECL(STATIC void, initiate_descriptor_downloads,
+          (const routerstatus_t *source, int purpose, smartlist_t *digests,
+           int lo, int hi, int pds_flags));
+STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
+                                          int serverdesc, int microdesc);
+
+#endif /* defined(ROUTERLIST_PRIVATE) */
+
+#endif /* !defined(TOR_ROUTERLIST_H) */
diff --git a/src/feature/nodelist/routerlist_st.h b/src/feature/nodelist/routerlist_st.h
new file mode 100644
index 0000000000..0b94a4dfcd
--- /dev/null
+++ b/src/feature/nodelist/routerlist_st.h
@@ -0,0 +1,40 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ROUTERLIST_ST_H
+#define ROUTERLIST_ST_H
+
+#include "or/desc_store_st.h"
+
+/** Contents of a directory of onion routers. */
+struct routerlist_t {
+  /** Map from server identity digest to a member of routers. */
+  struct digest_ri_map_t *identity_map;
+  /** Map from server descriptor digest to a signed_descriptor_t from
+   * routers or old_routers. */
+  struct digest_sd_map_t *desc_digest_map;
+  /** Map from extra-info digest to an extrainfo_t.  Only exists for
+   * routers in routers or old_routers. */
+  struct digest_ei_map_t *extra_info_map;
+  /** Map from extra-info digests to a signed_descriptor_t for a router
+   * descriptor having that extra-info digest.  Only exists for
+   * routers in routers or old_routers. */
+  struct digest_sd_map_t *desc_by_eid_map;
+  /** List of routerinfo_t for all currently live routers we know. */
+  smartlist_t *routers;
+  /** List of signed_descriptor_t for older router descriptors we're
+   * caching. */
+  smartlist_t *old_routers;
+  /** Store holding server descriptors.  If present, any router whose
+   * cache_info.saved_location == SAVED_IN_CACHE is stored in this file
+   * starting at cache_info.saved_offset */
+  desc_store_t desc_store;
+  /** Store holding extra-info documents. */
+  desc_store_t extrainfo_store;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c
new file mode 100644
index 0000000000..273666046b
--- /dev/null
+++ b/src/feature/nodelist/routerparse.c
@@ -0,0 +1,5693 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerparse.c
+ * \brief Code to parse and validate router descriptors, consenus directories,
+ *   and similar objects.
+ *
+ * The objects parsed by this module use a common text-based metaformat,
+ * documented in dir-spec.txt in torspec.git.  This module is itself divided
+ * into two major kinds of function: code to handle the metaformat, and code
+ * to convert from particular instances of the metaformat into the
+ * objects that Tor uses.
+ *
+ * The generic parsing code works by calling a table-based tokenizer on the
+ * input string.  Each token corresponds to a single line with a token, plus
+ * optional arguments on that line, plus an optional base-64 encoded object
+ * after that line.  Each token has a definition in a table of token_rule_t
+ * entries that describes how many arguments it can take, whether it takes an
+ * object, how many times it may appear, whether it must appear first, and so
+ * on.
+ *
+ * The tokenizer function tokenize_string() converts its string input into a
+ * smartlist full of instances of directory_token_t, according to a provided
+ * table of token_rule_t.
+ *
+ * The generic parts of this module additionally include functions for
+ * finding the start and end of signed information inside a signed object, and
+ * computing the digest that will be signed.
+ *
+ * There are also functions for saving objects to disk that have caused
+ * parsing to fail.
+ *
+ * The specific parts of this module describe conversions between
+ * particular lists of directory_token_t and particular objects.  The
+ * kinds of objects that can be parsed here are:
+ *  <ul>
+ *  <li>router descriptors (managed from routerlist.c)
+ *  <li>extra-info documents (managed from routerlist.c)
+ *  <li>microdescriptors (managed from microdesc.c)
+ *  <li>vote and consensus networkstatus documents, and the routerstatus_t
+ *    objects that they comprise (managed from networkstatus.c)
+ *  <li>detached-signature objects used by authorities for gathering
+ *    signatures on the networkstatus consensus (managed from dirvote.c)
+ *  <li>authority key certificates (managed from routerlist.c)
+ *  <li>hidden service descriptors (managed from rendcommon.c and rendcache.c)
+ * </ul>
+ *
+ * For no terribly good reason, the functions to <i>generate</i> signatures on
+ * the above directory objects are also in this module.
+ **/
+
+#define ROUTERPARSE_PRIVATE
+
+#include "or/or.h"
+#include "or/circuitstats.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/dirauth/shared_random.h"
+#include "or/dirserv.h"
+#include "or/entrynodes.h"
+#include "lib/memarea/memarea.h"
+#include "or/microdesc.h"
+#include "or/networkstatus.h"
+#include "or/parsecommon.h"
+#include "or/policies.h"
+#include "or/protover.h"
+#include "or/rendcommon.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerkeys.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "lib/sandbox/sandbox.h"
+#include "or/shared_random_client.h"
+#include "or/torcert.h"
+#include "or/voting_schedule.h"
+
+#include "or/dirauth/dirvote.h"
+
+#include "or/addr_policy_st.h"
+#include "or/authority_cert_st.h"
+#include "or/document_signature_st.h"
+#include "or/extend_info_st.h"
+#include "or/extrainfo_st.h"
+#include "or/microdesc_st.h"
+#include "or/networkstatus_st.h"
+#include "or/networkstatus_voter_info_st.h"
+#include "or/ns_detached_signatures_st.h"
+#include "or/rend_authorized_client_st.h"
+#include "or/rend_intro_point_st.h"
+#include "or/rend_service_descriptor_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerlist_st.h"
+#include "or/tor_version_st.h"
+#include "or/vote_microdesc_hash_st.h"
+#include "or/vote_routerstatus_st.h"
+
+#include "lib/container/bloomfilt.h"
+
+#undef log
+#include <math.h>
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+/****************************************************************************/
+
+/** List of tokens recognized in router descriptors */
+static token_rule_t routerdesc_token_table[] = {
+  T0N("reject",              K_REJECT,              ARGS,    NO_OBJ ),
+  T0N("accept",              K_ACCEPT,              ARGS,    NO_OBJ ),
+  T0N("reject6",             K_REJECT6,             ARGS,    NO_OBJ ),
+  T0N("accept6",             K_ACCEPT6,             ARGS,    NO_OBJ ),
+  T1_START( "router",        K_ROUTER,              GE(5),   NO_OBJ ),
+  T01("ipv6-policy",         K_IPV6_POLICY,         CONCAT_ARGS, NO_OBJ),
+  T1( "signing-key",         K_SIGNING_KEY,         NO_ARGS, NEED_KEY_1024 ),
+  T1( "onion-key",           K_ONION_KEY,           NO_ARGS, NEED_KEY_1024 ),
+  T01("ntor-onion-key",      K_ONION_KEY_NTOR,      GE(1), NO_OBJ ),
+  T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
+  T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
+  T01("uptime",              K_UPTIME,              GE(1),   NO_OBJ ),
+  T01("fingerprint",         K_FINGERPRINT,     CONCAT_ARGS, NO_OBJ ),
+  T01("hibernating",         K_HIBERNATING,         GE(1),   NO_OBJ ),
+  T01("platform",            K_PLATFORM,        CONCAT_ARGS, NO_OBJ ),
+  T01("proto",               K_PROTO,           CONCAT_ARGS, NO_OBJ ),
+  T01("contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
+  T01("read-history",        K_READ_HISTORY,        ARGS,    NO_OBJ ),
+  T01("write-history",       K_WRITE_HISTORY,       ARGS,    NO_OBJ ),
+  T01("extra-info-digest",   K_EXTRA_INFO_DIGEST,   GE(1),   NO_OBJ ),
+  T01("hidden-service-dir",  K_HIDDEN_SERVICE_DIR,  NO_ARGS, NO_OBJ ),
+  T01("identity-ed25519",    K_IDENTITY_ED25519,    NO_ARGS, NEED_OBJ ),
+  T01("master-key-ed25519",  K_MASTER_KEY_ED25519,  GE(1),   NO_OBJ ),
+  T01("router-sig-ed25519",  K_ROUTER_SIG_ED25519,  GE(1),   NO_OBJ ),
+  T01("onion-key-crosscert", K_ONION_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),
+  T01("ntor-onion-key-crosscert", K_NTOR_ONION_KEY_CROSSCERT,
+                                                    EQ(1),   NEED_OBJ ),
+
+  T01("allow-single-hop-exits",K_ALLOW_SINGLE_HOP_EXITS,    NO_ARGS, NO_OBJ ),
+
+  T01("family",              K_FAMILY,              ARGS,    NO_OBJ ),
+  T01("caches-extra-info",   K_CACHES_EXTRA_INFO,   NO_ARGS, NO_OBJ ),
+  T0N("or-address",          K_OR_ADDRESS,          GE(1),   NO_OBJ ),
+
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  T1( "bandwidth",           K_BANDWIDTH,           GE(3),   NO_OBJ ),
+  A01("@purpose",            A_PURPOSE,             GE(1),   NO_OBJ ),
+  T01("tunnelled-dir-server",K_DIR_TUNNELLED,       NO_ARGS, NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in extra-info documents. */
+static token_rule_t extrainfo_token_table[] = {
+  T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
+  T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
+  T01("identity-ed25519",    K_IDENTITY_ED25519,    NO_ARGS, NEED_OBJ ),
+  T01("router-sig-ed25519",  K_ROUTER_SIG_ED25519,  GE(1),   NO_OBJ ),
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  T01("read-history",        K_READ_HISTORY,        ARGS,    NO_OBJ ),
+  T01("write-history",       K_WRITE_HISTORY,       ARGS,    NO_OBJ ),
+  T01("dirreq-stats-end",    K_DIRREQ_END,          ARGS,    NO_OBJ ),
+  T01("dirreq-v2-ips",       K_DIRREQ_V2_IPS,       ARGS,    NO_OBJ ),
+  T01("dirreq-v3-ips",       K_DIRREQ_V3_IPS,       ARGS,    NO_OBJ ),
+  T01("dirreq-v2-reqs",      K_DIRREQ_V2_REQS,      ARGS,    NO_OBJ ),
+  T01("dirreq-v3-reqs",      K_DIRREQ_V3_REQS,      ARGS,    NO_OBJ ),
+  T01("dirreq-v2-share",     K_DIRREQ_V2_SHARE,     ARGS,    NO_OBJ ),
+  T01("dirreq-v3-share",     K_DIRREQ_V3_SHARE,     ARGS,    NO_OBJ ),
+  T01("dirreq-v2-resp",      K_DIRREQ_V2_RESP,      ARGS,    NO_OBJ ),
+  T01("dirreq-v3-resp",      K_DIRREQ_V3_RESP,      ARGS,    NO_OBJ ),
+  T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR,       ARGS,    NO_OBJ ),
+  T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR,       ARGS,    NO_OBJ ),
+  T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN,     ARGS,    NO_OBJ ),
+  T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN,     ARGS,    NO_OBJ ),
+  T01("entry-stats-end",     K_ENTRY_END,           ARGS,    NO_OBJ ),
+  T01("entry-ips",           K_ENTRY_IPS,           ARGS,    NO_OBJ ),
+  T01("cell-stats-end",      K_CELL_END,            ARGS,    NO_OBJ ),
+  T01("cell-processed-cells", K_CELL_PROCESSED,     ARGS,    NO_OBJ ),
+  T01("cell-queued-cells",   K_CELL_QUEUED,         ARGS,    NO_OBJ ),
+  T01("cell-time-in-queue",  K_CELL_TIME,           ARGS,    NO_OBJ ),
+  T01("cell-circuits-per-decile", K_CELL_CIRCS,     ARGS,    NO_OBJ ),
+  T01("exit-stats-end",      K_EXIT_END,            ARGS,    NO_OBJ ),
+  T01("exit-kibibytes-written", K_EXIT_WRITTEN,     ARGS,    NO_OBJ ),
+  T01("exit-kibibytes-read", K_EXIT_READ,           ARGS,    NO_OBJ ),
+  T01("exit-streams-opened", K_EXIT_OPENED,         ARGS,    NO_OBJ ),
+
+  T1_START( "extra-info",          K_EXTRA_INFO,          GE(2),   NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the body part of v3 networkstatus
+ * documents. */
+static token_rule_t rtrstatus_token_table[] = {
+  T01("p",                   K_P,               CONCAT_ARGS, NO_OBJ ),
+  T1( "r",                   K_R,                   GE(7),   NO_OBJ ),
+  T0N("a",                   K_A,                   GE(1),   NO_OBJ ),
+  T1( "s",                   K_S,                   ARGS,    NO_OBJ ),
+  T01("v",                   K_V,               CONCAT_ARGS, NO_OBJ ),
+  T01("w",                   K_W,                   ARGS,    NO_OBJ ),
+  T0N("m",                   K_M,               CONCAT_ARGS, NO_OBJ ),
+  T0N("id",                  K_ID,                  GE(2),   NO_OBJ ),
+  T01("pr",                  K_PROTO,           CONCAT_ARGS, NO_OBJ ),
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  END_OF_TABLE
+};
+
+/** List of tokens common to V3 authority certificates and V3 consensuses. */
+#define CERTIFICATE_MEMBERS                                                  \
+  T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION,           \
+                                                     GE(1),       NO_OBJ ),  \
+  T1("dir-identity-key", K_DIR_IDENTITY_KEY,         NO_ARGS,     NEED_KEY ),\
+  T1("dir-key-published",K_DIR_KEY_PUBLISHED,        CONCAT_ARGS, NO_OBJ),   \
+  T1("dir-key-expires",  K_DIR_KEY_EXPIRES,          CONCAT_ARGS, NO_OBJ),   \
+  T1("dir-signing-key",  K_DIR_SIGNING_KEY,          NO_ARGS,     NEED_KEY ),\
+  T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT,       NO_ARGS,     NEED_OBJ ),\
+  T1("dir-key-certification", K_DIR_KEY_CERTIFICATION,                       \
+                                                     NO_ARGS,     NEED_OBJ), \
+  T01("dir-address",     K_DIR_ADDRESS,              GE(1),       NO_OBJ),
+
+/** List of tokens recognized in V3 authority certificates. */
+static token_rule_t dir_key_certificate_table[] = {
+  CERTIFICATE_MEMBERS
+  T1("fingerprint",      K_FINGERPRINT,              CONCAT_ARGS, NO_OBJ ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in rendezvous service descriptors */
+static token_rule_t desc_token_table[] = {
+  T1_START("rendezvous-service-descriptor", R_RENDEZVOUS_SERVICE_DESCRIPTOR,
+           EQ(1), NO_OBJ),
+  T1("version", R_VERSION, EQ(1), NO_OBJ),
+  T1("permanent-key", R_PERMANENT_KEY, NO_ARGS, NEED_KEY_1024),
+  T1("secret-id-part", R_SECRET_ID_PART, EQ(1), NO_OBJ),
+  T1("publication-time", R_PUBLICATION_TIME, CONCAT_ARGS, NO_OBJ),
+  T1("protocol-versions", R_PROTOCOL_VERSIONS, EQ(1), NO_OBJ),
+  T01("introduction-points", R_INTRODUCTION_POINTS, NO_ARGS, NEED_OBJ),
+  T1_END("signature", R_SIGNATURE, NO_ARGS, NEED_OBJ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the (encrypted) list of introduction points of
+ * rendezvous service descriptors */
+static token_rule_t ipo_token_table[] = {
+  T1_START("introduction-point", R_IPO_IDENTIFIER, EQ(1), NO_OBJ),
+  T1("ip-address", R_IPO_IP_ADDRESS, EQ(1), NO_OBJ),
+  T1("onion-port", R_IPO_ONION_PORT, EQ(1), NO_OBJ),
+  T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024),
+  T1("service-key", R_IPO_SERVICE_KEY, NO_ARGS, NEED_KEY_1024),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the (possibly encrypted) list of introduction
+ * points of rendezvous service descriptors */
+static token_rule_t client_keys_token_table[] = {
+  T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+  T1("descriptor-cookie", C_DESCRIPTOR_COOKIE, EQ(1), NO_OBJ),
+  T01("client-key", C_CLIENT_KEY, NO_ARGS, NEED_SKEY_1024),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in V3 networkstatus votes. */
+static token_rule_t networkstatus_token_table[] = {
+  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
+                                                   GE(1),       NO_OBJ ),
+  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
+  T1("published",              K_PUBLISHED,        CONCAT_ARGS, NO_OBJ ),
+  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
+  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
+  T1("known-flags",            K_KNOWN_FLAGS,      ARGS,        NO_OBJ ),
+  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
+  T( "fingerprint",            K_FINGERPRINT,      CONCAT_ARGS, NO_OBJ ),
+  T01("signing-ed25519",       K_SIGNING_CERT_ED,  NO_ARGS ,    NEED_OBJ ),
+  T01("shared-rand-participate",K_SR_FLAG,         NO_ARGS,     NO_OBJ ),
+  T0N("shared-rand-commit",    K_COMMIT,           GE(3),       NO_OBJ ),
+  T01("shared-rand-previous-value", K_PREVIOUS_SRV,EQ(2),       NO_OBJ ),
+  T01("shared-rand-current-value",  K_CURRENT_SRV, EQ(2),       NO_OBJ ),
+  T0N("package",               K_PACKAGE,          CONCAT_ARGS, NO_OBJ ),
+  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+
+  CERTIFICATE_MEMBERS
+
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  T1( "contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
+  T1( "dir-source",          K_DIR_SOURCE,      GE(6),       NO_OBJ ),
+  T01("legacy-dir-key",      K_LEGACY_DIR_KEY,  GE(1),       NO_OBJ ),
+  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
+  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T1( "consensus-methods",   K_CONSENSUS_METHODS, GE(1),     NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in V3 networkstatus consensuses. */
+static token_rule_t networkstatus_consensus_token_table[] = {
+  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
+                                                   GE(1),       NO_OBJ ),
+  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
+  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
+  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
+
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+
+  T1N("dir-source",          K_DIR_SOURCE,          GE(6),   NO_OBJ ),
+  T1N("contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
+  T1N("vote-digest",         K_VOTE_DIGEST,         GE(1),   NO_OBJ ),
+
+  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
+
+  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T01("consensus-method",    K_CONSENSUS_METHOD,    EQ(1),   NO_OBJ),
+  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
+
+  T01("shared-rand-previous-value", K_PREVIOUS_SRV, EQ(2),   NO_OBJ ),
+  T01("shared-rand-current-value",  K_CURRENT_SRV,  EQ(2),   NO_OBJ ),
+
+  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the footer of v1 directory footers. */
+static token_rule_t networkstatus_vote_footer_token_table[] = {
+  T01("directory-footer",    K_DIRECTORY_FOOTER,    NO_ARGS,   NO_OBJ ),
+  T01("bandwidth-weights",   K_BW_WEIGHTS,          ARGS,      NO_OBJ ),
+  T(  "directory-signature", K_DIRECTORY_SIGNATURE, GE(2),     NEED_OBJ ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in detached networkstatus signature documents. */
+static token_rule_t networkstatus_detached_signature_token_table[] = {
+  T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1),       NO_OBJ ),
+  T("additional-digest",       K_ADDITIONAL_DIGEST,GE(3),       NO_OBJ ),
+  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
+  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T("additional-signature",  K_ADDITIONAL_SIGNATURE, GE(4),   NEED_OBJ ),
+  T1N("directory-signature", K_DIRECTORY_SIGNATURE,  GE(2),   NEED_OBJ ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in microdescriptors */
+static token_rule_t microdesc_token_table[] = {
+  T1_START("onion-key",        K_ONION_KEY,        NO_ARGS,     NEED_KEY_1024),
+  T01("ntor-onion-key",        K_ONION_KEY_NTOR,   GE(1),       NO_OBJ ),
+  T0N("id",                    K_ID,               GE(2),       NO_OBJ ),
+  T0N("a",                     K_A,                GE(1),       NO_OBJ ),
+  T01("family",                K_FAMILY,           ARGS,        NO_OBJ ),
+  T01("p",                     K_P,                CONCAT_ARGS, NO_OBJ ),
+  T01("p6",                    K_P6,               CONCAT_ARGS, NO_OBJ ),
+  A01("@last-listed",          A_LAST_LISTED,      CONCAT_ARGS, NO_OBJ ),
+  END_OF_TABLE
+};
+
+#undef T
+
+/* static function prototypes */
+static int router_add_exit_policy(routerinfo_t *router,directory_token_t *tok);
+static addr_policy_t *router_parse_addr_policy(directory_token_t *tok,
+                                               unsigned fmt_flags);
+static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok);
+
+static int router_get_hash_impl_helper(const char *s, size_t s_len,
+                            const char *start_str,
+                            const char *end_str, char end_c,
+                            int log_severity,
+                            const char **start_out, const char **end_out);
+static int router_get_hash_impl(const char *s, size_t s_len, char *digest,
+                                const char *start_str, const char *end_str,
+                                char end_char,
+                                digest_algorithm_t alg);
+static int router_get_hashes_impl(const char *s, size_t s_len,
+                                  common_digests_t *digests,
+                                  const char *start_str, const char *end_str,
+                                  char end_char);
+static smartlist_t *find_all_exitpolicy(smartlist_t *s);
+
+#define CST_NO_CHECK_OBJTYPE  (1<<0)
+static int check_signature_token(const char *digest,
+                                 ssize_t digest_len,
+                                 directory_token_t *tok,
+                                 crypto_pk_t *pkey,
+                                 int flags,
+                                 const char *doctype);
+
+#undef DEBUG_AREA_ALLOC
+
+#ifdef DEBUG_AREA_ALLOC
+#define DUMP_AREA(a,name) STMT_BEGIN                              \
+  size_t alloc=0, used=0;                                         \
+  memarea_get_stats((a),&alloc,&used);                            \
+  log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.",   \
+            name, (unsigned long)alloc, (unsigned long)used);     \
+  STMT_END
+#else /* !(defined(DEBUG_AREA_ALLOC)) */
+#define DUMP_AREA(a,name) STMT_NIL
+#endif /* defined(DEBUG_AREA_ALLOC) */
+
+/* Dump mechanism for unparseable descriptors */
+
+/** List of dumped descriptors for FIFO cleanup purposes */
+STATIC smartlist_t *descs_dumped = NULL;
+/** Total size of dumped descriptors for FIFO cleanup */
+STATIC uint64_t len_descs_dumped = 0;
+/** Directory to stash dumps in */
+static int have_dump_desc_dir = 0;
+static int problem_with_dump_desc_dir = 0;
+
+#define DESC_DUMP_DATADIR_SUBDIR "unparseable-descs"
+#define DESC_DUMP_BASE_FILENAME "unparseable-desc"
+
+/** Find the dump directory and check if we'll be able to create it */
+static void
+dump_desc_init(void)
+{
+  char *dump_desc_dir;
+
+  dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
+
+  /*
+   * We just check for it, don't create it at this point; we'll
+   * create it when we need it if it isn't already there.
+   */
+  if (check_private_dir(dump_desc_dir, CPD_CHECK, get_options()->User) < 0) {
+    /* Error, log and flag it as having a problem */
+    log_notice(LD_DIR,
+               "Doesn't look like we'll be able to create descriptor dump "
+               "directory %s; dumps will be disabled.",
+               dump_desc_dir);
+    problem_with_dump_desc_dir = 1;
+    tor_free(dump_desc_dir);
+    return;
+  }
+
+  /* Check if it exists */
+  switch (file_status(dump_desc_dir)) {
+    case FN_DIR:
+      /* We already have a directory */
+      have_dump_desc_dir = 1;
+      break;
+    case FN_NOENT:
+      /* Nothing, we'll need to create it later */
+      have_dump_desc_dir = 0;
+      break;
+    case FN_ERROR:
+      /* Log and flag having a problem */
+      log_notice(LD_DIR,
+                 "Couldn't check whether descriptor dump directory %s already"
+                 " exists: %s",
+                 dump_desc_dir, strerror(errno));
+      problem_with_dump_desc_dir = 1;
+      break;
+    case FN_FILE:
+    case FN_EMPTY:
+    default:
+      /* Something else was here! */
+      log_notice(LD_DIR,
+                 "Descriptor dump directory %s already exists and isn't a "
+                 "directory",
+                 dump_desc_dir);
+      problem_with_dump_desc_dir = 1;
+  }
+
+  if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
+    dump_desc_populate_fifo_from_directory(dump_desc_dir);
+  }
+
+  tor_free(dump_desc_dir);
+}
+
+/** Create the dump directory if needed and possible */
+static void
+dump_desc_create_dir(void)
+{
+  char *dump_desc_dir;
+
+  /* If the problem flag is set, skip it */
+  if (problem_with_dump_desc_dir) return;
+
+  /* Do we need it? */
+  if (!have_dump_desc_dir) {
+    dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
+
+    if (check_private_dir(dump_desc_dir, CPD_CREATE,
+                          get_options()->User) < 0) {
+      log_notice(LD_DIR,
+                 "Failed to create descriptor dump directory %s",
+                 dump_desc_dir);
+      problem_with_dump_desc_dir = 1;
+    }
+
+    /* Okay, we created it */
+    have_dump_desc_dir = 1;
+
+    tor_free(dump_desc_dir);
+  }
+}
+
+/** Dump desc FIFO/cleanup; take ownership of the given filename, add it to
+ * the FIFO, and clean up the oldest entries to the extent they exceed the
+ * configured cap.  If any old entries with a matching hash existed, they
+ * just got overwritten right before this was called and we should adjust
+ * the total size counter without deleting them.
+ */
+static void
+dump_desc_fifo_add_and_clean(char *filename, const uint8_t *digest_sha256,
+                             size_t len)
+{
+  dumped_desc_t *ent = NULL, *tmp;
+  uint64_t max_len;
+
+  tor_assert(filename != NULL);
+  tor_assert(digest_sha256 != NULL);
+
+  if (descs_dumped == NULL) {
+    /* We better have no length, then */
+    tor_assert(len_descs_dumped == 0);
+    /* Make a smartlist */
+    descs_dumped = smartlist_new();
+  }
+
+  /* Make a new entry to put this one in */
+  ent = tor_malloc_zero(sizeof(*ent));
+  ent->filename = filename;
+  ent->len = len;
+  ent->when = time(NULL);
+  memcpy(ent->digest_sha256, digest_sha256, DIGEST256_LEN);
+
+  /* Do we need to do some cleanup? */
+  max_len = get_options()->MaxUnparseableDescSizeToLog;
+  /* Iterate over the list until we've freed enough space */
+  while (len > max_len - len_descs_dumped &&
+         smartlist_len(descs_dumped) > 0) {
+    /* Get the oldest thing on the list */
+    tmp = (dumped_desc_t *)(smartlist_get(descs_dumped, 0));
+
+    /*
+     * Check if it matches the filename we just added, so we don't delete
+     * something we just emitted if we get repeated identical descriptors.
+     */
+    if (strcmp(tmp->filename, filename) != 0) {
+      /* Delete it and adjust the length counter */
+      tor_unlink(tmp->filename);
+      tor_assert(len_descs_dumped >= tmp->len);
+      len_descs_dumped -= tmp->len;
+      log_info(LD_DIR,
+               "Deleting old unparseable descriptor dump %s due to "
+               "space limits",
+               tmp->filename);
+    } else {
+      /*
+       * Don't delete, but do adjust the counter since we will bump it
+       * later
+       */
+      tor_assert(len_descs_dumped >= tmp->len);
+      len_descs_dumped -= tmp->len;
+      log_info(LD_DIR,
+               "Replacing old descriptor dump %s with new identical one",
+               tmp->filename);
+    }
+
+    /* Free it and remove it from the list */
+    smartlist_del_keeporder(descs_dumped, 0);
+    tor_free(tmp->filename);
+    tor_free(tmp);
+  }
+
+  /* Append our entry to the end of the list and bump the counter */
+  smartlist_add(descs_dumped, ent);
+  len_descs_dumped += len;
+}
+
+/** Check if we already have a descriptor for this hash and move it to the
+ * head of the queue if so.  Return 1 if one existed and 0 otherwise.
+ */
+static int
+dump_desc_fifo_bump_hash(const uint8_t *digest_sha256)
+{
+  dumped_desc_t *match = NULL;
+
+  tor_assert(digest_sha256);
+
+  if (descs_dumped) {
+    /* Find a match if one exists */
+    SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
+      if (ent &&
+          tor_memeq(ent->digest_sha256, digest_sha256, DIGEST256_LEN)) {
+        /*
+         * Save a pointer to the match and remove it from its current
+         * position.
+         */
+        match = ent;
+        SMARTLIST_DEL_CURRENT_KEEPORDER(descs_dumped, ent);
+        break;
+      }
+    } SMARTLIST_FOREACH_END(ent);
+
+    if (match) {
+      /* Update the timestamp */
+      match->when = time(NULL);
+      /* Add it back at the end of the list */
+      smartlist_add(descs_dumped, match);
+
+      /* Indicate we found one */
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+/** Clean up on exit; just memory, leave the dumps behind
+ */
+STATIC void
+dump_desc_fifo_cleanup(void)
+{
+  if (descs_dumped) {
+    /* Free each descriptor */
+    SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
+      tor_assert(ent);
+      tor_free(ent->filename);
+      tor_free(ent);
+    } SMARTLIST_FOREACH_END(ent);
+    /* Free the list */
+    smartlist_free(descs_dumped);
+    descs_dumped = NULL;
+    len_descs_dumped = 0;
+  }
+}
+
+/** Handle one file for dump_desc_populate_fifo_from_directory(); make sure
+ * the filename is sensibly formed and matches the file content, and either
+ * return a dumped_desc_t for it or remove the file and return NULL.
+ */
+MOCK_IMPL(STATIC dumped_desc_t *,
+dump_desc_populate_one_file, (const char *dirname, const char *f))
+{
+  dumped_desc_t *ent = NULL;
+  char *path = NULL, *desc = NULL;
+  const char *digest_str;
+  char digest[DIGEST256_LEN], content_digest[DIGEST256_LEN];
+  /* Expected prefix before digest in filenames */
+  const char *f_pfx = DESC_DUMP_BASE_FILENAME ".";
+  /*
+   * Stat while reading; this is important in case the file
+   * contains a NUL character.
+   */
+  struct stat st;
+
+  /* Sanity-check args */
+  tor_assert(dirname != NULL);
+  tor_assert(f != NULL);
+
+  /* Form the full path */
+  tor_asprintf(&path, "%s" PATH_SEPARATOR "%s", dirname, f);
+
+  /* Check that f has the form DESC_DUMP_BASE_FILENAME.<digest256> */
+
+  if (!strcmpstart(f, f_pfx)) {
+    /* It matches the form, but is the digest parseable as such? */
+    digest_str = f + strlen(f_pfx);
+    if (base16_decode(digest, DIGEST256_LEN,
+                      digest_str, strlen(digest_str)) != DIGEST256_LEN) {
+      /* We failed to decode it */
+      digest_str = NULL;
+    }
+  } else {
+    /* No match */
+    digest_str = NULL;
+  }
+
+  if (!digest_str) {
+    /* We couldn't get a sensible digest */
+    log_notice(LD_DIR,
+               "Removing unrecognized filename %s from unparseable "
+               "descriptors directory", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+  /*
+   * The filename has the form DESC_DUMP_BASE_FILENAME "." <digest256> and
+   * we've decoded the digest.  Next, check that we can read it and the
+   * content matches this digest.  We are relying on the fact that if the
+   * file contains a '\0', read_file_to_str() will allocate space for and
+   * read the entire file and return the correct size in st.
+   */
+  desc = read_file_to_str(path, RFTS_IGNORE_MISSING|RFTS_BIN, &st);
+  if (!desc) {
+    /* We couldn't read it */
+    log_notice(LD_DIR,
+               "Failed to read %s from unparseable descriptors directory; "
+               "attempting to remove it.", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+#if SIZE_MAX > UINT64_MAX
+  if (BUG((uint64_t)st.st_size > (uint64_t)SIZE_MAX)) {
+    /* LCOV_EXCL_START
+     * Should be impossible since RFTS above should have failed to read the
+     * huge file into RAM. */
+    goto done;
+    /* LCOV_EXCL_STOP */
+  }
+#endif /* SIZE_MAX > UINT64_MAX */
+  if (BUG(st.st_size < 0)) {
+    /* LCOV_EXCL_START
+     * Should be impossible, since the OS isn't supposed to be b0rken. */
+    goto done;
+    /* LCOV_EXCL_STOP */
+  }
+  /* (Now we can be sure that st.st_size is safe to cast to a size_t.) */
+
+  /*
+   * We got one; now compute its digest and check that it matches the
+   * filename.
+   */
+  if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size,
+                       DIGEST_SHA256) < 0) {
+    /* Weird, but okay */
+    log_info(LD_DIR,
+             "Unable to hash content of %s from unparseable descriptors "
+             "directory", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+  /* Compare the digests */
+  if (tor_memneq(digest, content_digest, DIGEST256_LEN)) {
+    /* No match */
+    log_info(LD_DIR,
+             "Hash of %s from unparseable descriptors directory didn't "
+             "match its filename; removing it", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+  /* Okay, it's a match, we should prepare ent */
+  ent = tor_malloc_zero(sizeof(dumped_desc_t));
+  ent->filename = path;
+  memcpy(ent->digest_sha256, digest, DIGEST256_LEN);
+  ent->len = (size_t) st.st_size;
+  ent->when = st.st_mtime;
+  /* Null out path so we don't free it out from under ent */
+  path = NULL;
+
+ done:
+  /* Free allocations if we had them */
+  tor_free(desc);
+  tor_free(path);
+
+  return ent;
+}
+
+/** Sort helper for dump_desc_populate_fifo_from_directory(); compares
+ * the when field of dumped_desc_ts in a smartlist to put the FIFO in
+ * the correct order after reconstructing it from the directory.
+ */
+static int
+dump_desc_compare_fifo_entries(const void **a_v, const void **b_v)
+{
+  const dumped_desc_t **a = (const dumped_desc_t **)a_v;
+  const dumped_desc_t **b = (const dumped_desc_t **)b_v;
+
+  if ((a != NULL) && (*a != NULL)) {
+    if ((b != NULL) && (*b != NULL)) {
+      /* We have sensible dumped_desc_ts to compare */
+      if ((*a)->when < (*b)->when) {
+        return -1;
+      } else if ((*a)->when == (*b)->when) {
+        return 0;
+      } else {
+        return 1;
+      }
+    } else {
+      /*
+       * We shouldn't see this, but what the hell, NULLs precede everythin
+       * else
+       */
+      return 1;
+    }
+  } else {
+    return -1;
+  }
+}
+
+/** Scan the contents of the directory, and update FIFO/counters; this will
+ * consistency-check descriptor dump filenames against hashes of descriptor
+ * dump file content, and remove any inconsistent/unreadable dumps, and then
+ * reconstruct the dump FIFO as closely as possible for the last time the
+ * tor process shut down.  If a previous dump was repeated more than once and
+ * moved ahead in the FIFO, the mtime will not have been updated and the
+ * reconstructed order will be wrong, but will always be a permutation of
+ * the original.
+ */
+STATIC void
+dump_desc_populate_fifo_from_directory(const char *dirname)
+{
+  smartlist_t *files = NULL;
+  dumped_desc_t *ent = NULL;
+
+  tor_assert(dirname != NULL);
+
+  /* Get a list of files */
+  files = tor_listdir(dirname);
+  if (!files) {
+    log_notice(LD_DIR,
+               "Unable to get contents of unparseable descriptor dump "
+               "directory %s",
+               dirname);
+    return;
+  }
+
+  /*
+   * Iterate through the list and decide which files should go in the
+   * FIFO and which should be purged.
+   */
+
+  SMARTLIST_FOREACH_BEGIN(files, char *, f) {
+    /* Try to get a FIFO entry */
+    ent = dump_desc_populate_one_file(dirname, f);
+    if (ent) {
+      /*
+       * We got one; add it to the FIFO.  No need for duplicate checking
+       * here since we just verified the name and digest match.
+       */
+
+      /* Make sure we have a list to add it to */
+      if (!descs_dumped) {
+        descs_dumped = smartlist_new();
+        len_descs_dumped = 0;
+      }
+
+      /* Add it and adjust the counter */
+      smartlist_add(descs_dumped, ent);
+      len_descs_dumped += ent->len;
+    }
+    /*
+     * If we didn't, we will have unlinked the file if necessary and
+     * possible, and emitted a log message about it, so just go on to
+     * the next.
+     */
+  } SMARTLIST_FOREACH_END(f);
+
+  /* Did we get anything? */
+  if (descs_dumped != NULL) {
+    /* Sort the FIFO in order of increasing timestamp */
+    smartlist_sort(descs_dumped, dump_desc_compare_fifo_entries);
+
+    /* Log some stats */
+    log_info(LD_DIR,
+             "Reloaded unparseable descriptor dump FIFO with %d dump(s) "
+             "totaling %"PRIu64 " bytes",
+             smartlist_len(descs_dumped), (len_descs_dumped));
+  }
+
+  /* Free the original list */
+  SMARTLIST_FOREACH(files, char *, f, tor_free(f));
+  smartlist_free(files);
+}
+
+/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of
+ * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more
+ * than one descriptor to disk per minute. If there is already such a
+ * file in the data directory, overwrite it. */
+MOCK_IMPL(STATIC void,
+dump_desc,(const char *desc, const char *type))
+{
+  tor_assert(desc);
+  tor_assert(type);
+  size_t len;
+  /* The SHA256 of the string */
+  uint8_t digest_sha256[DIGEST256_LEN];
+  char digest_sha256_hex[HEX_DIGEST256_LEN+1];
+  /* Filename to log it to */
+  char *debugfile, *debugfile_base;
+
+  /* Get the hash for logging purposes anyway */
+  len = strlen(desc);
+  if (crypto_digest256((char *)digest_sha256, desc, len,
+                       DIGEST_SHA256) < 0) {
+    log_info(LD_DIR,
+             "Unable to parse descriptor of type %s, and unable to even hash"
+             " it!", type);
+    goto err;
+  }
+
+  base16_encode(digest_sha256_hex, sizeof(digest_sha256_hex),
+                (const char *)digest_sha256, sizeof(digest_sha256));
+
+  /*
+   * We mention type and hash in the main log; don't clutter up the files
+   * with anything but the exact dump.
+   */
+  tor_asprintf(&debugfile_base,
+               DESC_DUMP_BASE_FILENAME ".%s", digest_sha256_hex);
+  debugfile = get_datadir_fname2(DESC_DUMP_DATADIR_SUBDIR, debugfile_base);
+
+  /*
+   * Check if the sandbox is active or will become active; see comment
+   * below at the log message for why.
+   */
+  if (!(sandbox_is_active() || get_options()->Sandbox)) {
+    if (len <= get_options()->MaxUnparseableDescSizeToLog) {
+      if (!dump_desc_fifo_bump_hash(digest_sha256)) {
+        /* Create the directory if needed */
+        dump_desc_create_dir();
+        /* Make sure we've got it */
+        if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
+          /* Write it, and tell the main log about it */
+          write_str_to_file(debugfile, desc, 1);
+          log_info(LD_DIR,
+                   "Unable to parse descriptor of type %s with hash %s and "
+                   "length %lu. See file %s in data directory for details.",
+                   type, digest_sha256_hex, (unsigned long)len,
+                   debugfile_base);
+          dump_desc_fifo_add_and_clean(debugfile, digest_sha256, len);
+          /* Since we handed ownership over, don't free debugfile later */
+          debugfile = NULL;
+        } else {
+          /* Problem with the subdirectory */
+          log_info(LD_DIR,
+                   "Unable to parse descriptor of type %s with hash %s and "
+                   "length %lu. Descriptor not dumped because we had a "
+                   "problem creating the " DESC_DUMP_DATADIR_SUBDIR
+                   " subdirectory",
+                   type, digest_sha256_hex, (unsigned long)len);
+          /* We do have to free debugfile in this case */
+        }
+      } else {
+        /* We already had one with this hash dumped */
+        log_info(LD_DIR,
+                 "Unable to parse descriptor of type %s with hash %s and "
+                 "length %lu. Descriptor not dumped because one with that "
+                 "hash has already been dumped.",
+                 type, digest_sha256_hex, (unsigned long)len);
+        /* We do have to free debugfile in this case */
+      }
+    } else {
+      /* Just log that it happened without dumping */
+      log_info(LD_DIR,
+               "Unable to parse descriptor of type %s with hash %s and "
+               "length %lu. Descriptor not dumped because it exceeds maximum"
+               " log size all by itself.",
+               type, digest_sha256_hex, (unsigned long)len);
+      /* We do have to free debugfile in this case */
+    }
+  } else {
+    /*
+     * Not logging because the sandbox is active and seccomp2 apparently
+     * doesn't have a sensible way to allow filenames according to a pattern
+     * match.  (If we ever figure out how to say "allow writes to /regex/",
+     * remove this checK).
+     */
+    log_info(LD_DIR,
+             "Unable to parse descriptor of type %s with hash %s and "
+             "length %lu. Descriptor not dumped because the sandbox is "
+             "configured",
+             type, digest_sha256_hex, (unsigned long)len);
+  }
+
+  tor_free(debugfile_base);
+  tor_free(debugfile);
+
+ err:
+  return;
+}
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in
+ * <b>s</b>.  Return 0 on success, -1 on failure.
+ */
+int
+router_get_dir_hash(const char *s, char *digest)
+{
+  return router_get_hash_impl(s, strlen(s), digest,
+                              "signed-directory","\ndirectory-signature",'\n',
+                              DIGEST_SHA1);
+}
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in
+ * <b>s</b>. Return 0 on success, -1 on failure.
+ */
+int
+router_get_router_hash(const char *s, size_t s_len, char *digest)
+{
+  return router_get_hash_impl(s, s_len, digest,
+                              "router ","\nrouter-signature", '\n',
+                              DIGEST_SHA1);
+}
+
+/** Try to find the start and end of the signed portion of a networkstatus
+ * document in <b>s</b>. On success, set <b>start_out</b> to the first
+ * character of the document, and <b>end_out</b> to a position one after the
+ * final character of the signed document, and return 0.  On failure, return
+ * -1. */
+int
+router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                              const char **start_out,
+                                              const char **end_out)
+{
+  return router_get_hash_impl_helper(s, strlen(s),
+                                     "network-status-version",
+                                     "\ndirectory-signature",
+                                     ' ', LOG_INFO,
+                                     start_out, end_out);
+}
+
+/** Set <b>digest_out</b> to the SHA3-256 digest of the signed portion of the
+ * networkstatus vote in <b>s</b> -- or of the entirety of <b>s</b> if no
+ * signed portion can be identified.  Return 0 on success, -1 on failure. */
+int
+router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                           const char *s)
+{
+  const char *start, *end;
+  if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
+    start = s;
+    end = s + strlen(s);
+  }
+  tor_assert(start);
+  tor_assert(end);
+  return crypto_digest256((char*)digest_out, start, end-start,
+                          DIGEST_SHA3_256);
+}
+
+/** Set <b>digests</b> to all the digests of the consensus document in
+ * <b>s</b> */
+int
+router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
+{
+  return router_get_hashes_impl(s,strlen(s),digests,
+                                "network-status-version",
+                                "\ndirectory-signature",
+                                ' ');
+}
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the <b>s_len</b>-byte
+ * extrainfo string at <b>s</b>.  Return 0 on success, -1 on failure. */
+int
+router_get_extrainfo_hash(const char *s, size_t s_len, char *digest)
+{
+  return router_get_hash_impl(s, s_len, digest, "extra-info",
+                              "\nrouter-signature",'\n', DIGEST_SHA1);
+}
+
+/** Helper: used to generate signatures for routers, directories and
+ * network-status objects.  Given a <b>digest_len</b>-byte digest in
+ * <b>digest</b> and a secret <b>private_key</b>, generate an PKCS1-padded
+ * signature, BASE64-encode it, surround it with -----BEGIN/END----- pairs,
+ * and return the new signature on success or NULL on failure.
+ */
+char *
+router_get_dirobj_signature(const char *digest,
+                            size_t digest_len,
+                            const crypto_pk_t *private_key)
+{
+  char *signature;
+  size_t i, keysize;
+  int siglen;
+  char *buf = NULL;
+  size_t buf_len;
+  /* overestimate of BEGIN/END lines total len. */
+#define BEGIN_END_OVERHEAD_LEN 64
+
+  keysize = crypto_pk_keysize(private_key);
+  signature = tor_malloc(keysize);
+  siglen = crypto_pk_private_sign(private_key, signature, keysize,
+                                  digest, digest_len);
+  if (siglen < 0) {
+    log_warn(LD_BUG,"Couldn't sign digest.");
+    goto err;
+  }
+
+  /* The *2 here is a ridiculous overestimate of base-64 overhead. */
+  buf_len = (siglen * 2) + BEGIN_END_OVERHEAD_LEN;
+  buf = tor_malloc(buf_len);
+
+  if (strlcpy(buf, "-----BEGIN SIGNATURE-----\n", buf_len) >= buf_len)
+    goto truncated;
+
+  i = strlen(buf);
+  if (base64_encode(buf+i, buf_len-i, signature, siglen,
+                    BASE64_ENCODE_MULTILINE) < 0) {
+    log_warn(LD_BUG,"couldn't base64-encode signature");
+    goto err;
+  }
+
+  if (strlcat(buf, "-----END SIGNATURE-----\n", buf_len) >= buf_len)
+    goto truncated;
+
+  tor_free(signature);
+  return buf;
+
+ truncated:
+  log_warn(LD_BUG,"tried to exceed string length.");
+ err:
+  tor_free(signature);
+  tor_free(buf);
+  return NULL;
+}
+
+/** Helper: used to generate signatures for routers, directories and
+ * network-status objects.  Given a digest in <b>digest</b> and a secret
+ * <b>private_key</b>, generate a PKCS1-padded signature, BASE64-encode it,
+ * surround it with -----BEGIN/END----- pairs, and write it to the
+ * <b>buf_len</b>-byte buffer at <b>buf</b>.  Return 0 on success, -1 on
+ * failure.
+ */
+int
+router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest,
+                               size_t digest_len, crypto_pk_t *private_key)
+{
+  size_t sig_len, s_len;
+  char *sig = router_get_dirobj_signature(digest, digest_len, private_key);
+  if (!sig) {
+    log_warn(LD_BUG, "No signature generated");
+    return -1;
+  }
+  sig_len = strlen(sig);
+  s_len = strlen(buf);
+  if (sig_len + s_len + 1 > buf_len) {
+    log_warn(LD_BUG, "Not enough room for signature");
+    tor_free(sig);
+    return -1;
+  }
+  memcpy(buf+s_len, sig, sig_len+1);
+  tor_free(sig);
+  return 0;
+}
+
+/** Return VS_RECOMMENDED if <b>myversion</b> is contained in
+ * <b>versionlist</b>.  Else, return VS_EMPTY if versionlist has no
+ * entries. Else, return VS_OLD if every member of
+ * <b>versionlist</b> is newer than <b>myversion</b>.  Else, return
+ * VS_NEW_IN_SERIES if there is at least one member of <b>versionlist</b> in
+ * the same series (major.minor.micro) as <b>myversion</b>, but no such member
+ * is newer than <b>myversion.</b>.  Else, return VS_NEW if every member of
+ * <b>versionlist</b> is older than <b>myversion</b>.  Else, return
+ * VS_UNRECOMMENDED.
+ *
+ * (versionlist is a comma-separated list of version strings,
+ * optionally prefixed with "Tor".  Versions that can't be parsed are
+ * ignored.)
+ */
+version_status_t
+tor_version_is_obsolete(const char *myversion, const char *versionlist)
+{
+  tor_version_t mine, other;
+  int found_newer = 0, found_older = 0, found_newer_in_series = 0,
+    found_any_in_series = 0, r, same;
+  version_status_t ret = VS_UNRECOMMENDED;
+  smartlist_t *version_sl;
+
+  log_debug(LD_CONFIG,"Checking whether version '%s' is in '%s'",
+            myversion, versionlist);
+
+  if (tor_version_parse(myversion, &mine)) {
+    log_err(LD_BUG,"I couldn't parse my own version (%s)", myversion);
+    tor_assert(0);
+  }
+  version_sl = smartlist_new();
+  smartlist_split_string(version_sl, versionlist, ",", SPLIT_SKIP_SPACE, 0);
+
+  if (!strlen(versionlist)) { /* no authorities cared or agreed */
+    ret = VS_EMPTY;
+    goto done;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(version_sl, const char *, cp) {
+    if (!strcmpstart(cp, "Tor "))
+      cp += 4;
+
+    if (tor_version_parse(cp, &other)) {
+      /* Couldn't parse other; it can't be a match. */
+    } else {
+      same = tor_version_same_series(&mine, &other);
+      if (same)
+        found_any_in_series = 1;
+      r = tor_version_compare(&mine, &other);
+      if (r==0) {
+        ret = VS_RECOMMENDED;
+        goto done;
+      } else if (r<0) {
+        found_newer = 1;
+        if (same)
+          found_newer_in_series = 1;
+      } else if (r>0) {
+        found_older = 1;
+      }
+    }
+  } SMARTLIST_FOREACH_END(cp);
+
+  /* We didn't find the listed version. Is it new or old? */
+  if (found_any_in_series && !found_newer_in_series && found_newer) {
+    ret = VS_NEW_IN_SERIES;
+  } else if (found_newer && !found_older) {
+    ret = VS_OLD;
+  } else if (found_older && !found_newer) {
+    ret = VS_NEW;
+  } else {
+    ret = VS_UNRECOMMENDED;
+  }
+
+ done:
+  SMARTLIST_FOREACH(version_sl, char *, version, tor_free(version));
+  smartlist_free(version_sl);
+  return ret;
+}
+
+MOCK_IMPL(STATIC int,
+signed_digest_equals, (const uint8_t *d1, const uint8_t *d2, size_t len))
+{
+  return tor_memeq(d1, d2, len);
+}
+
+/** Check whether the object body of the token in <b>tok</b> has a good
+ * signature for <b>digest</b> using key <b>pkey</b>.
+ * If <b>CST_NO_CHECK_OBJTYPE</b> is set, do not check
+ * the object type of the signature object. Use <b>doctype</b> as the type of
+ * the document when generating log messages.  Return 0 on success, negative
+ * on failure.
+ */
+static int
+check_signature_token(const char *digest,
+                      ssize_t digest_len,
+                      directory_token_t *tok,
+                      crypto_pk_t *pkey,
+                      int flags,
+                      const char *doctype)
+{
+  char *signed_digest;
+  size_t keysize;
+  const int check_objtype = ! (flags & CST_NO_CHECK_OBJTYPE);
+
+  tor_assert(pkey);
+  tor_assert(tok);
+  tor_assert(digest);
+  tor_assert(doctype);
+
+  if (check_objtype) {
+    if (strcmp(tok->object_type, "SIGNATURE")) {
+      log_warn(LD_DIR, "Bad object type on %s signature", doctype);
+      return -1;
+    }
+  }
+
+  keysize = crypto_pk_keysize(pkey);
+  signed_digest = tor_malloc(keysize);
+  if (crypto_pk_public_checksig(pkey, signed_digest, keysize,
+                                tok->object_body, tok->object_size)
+      < digest_len) {
+    log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype);
+    tor_free(signed_digest);
+    return -1;
+  }
+  //  log_debug(LD_DIR,"Signed %s hash starts %s", doctype,
+  //            hex_str(signed_digest,4));
+  if (! signed_digest_equals((const uint8_t *)digest,
+                             (const uint8_t *)signed_digest, digest_len)) {
+    log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype);
+    tor_free(signed_digest);
+    return -1;
+  }
+  tor_free(signed_digest);
+  return 0;
+}
+
+/** Helper: move *<b>s_ptr</b> ahead to the next router, the next extra-info,
+ * or to the first of the annotations proceeding the next router or
+ * extra-info---whichever comes first.  Set <b>is_extrainfo_out</b> to true if
+ * we found an extrainfo, or false if found a router. Do not scan beyond
+ * <b>eos</b>.  Return -1 if we found nothing; 0 if we found something. */
+static int
+find_start_of_next_router_or_extrainfo(const char **s_ptr,
+                                       const char *eos,
+                                       int *is_extrainfo_out)
+{
+  const char *annotations = NULL;
+  const char *s = *s_ptr;
+
+  s = eat_whitespace_eos(s, eos);
+
+  while (s < eos-32) {  /* 32 gives enough room for a the first keyword. */
+    /* We're at the start of a line. */
+    tor_assert(*s != '\n');
+
+    if (*s == '@' && !annotations) {
+      annotations = s;
+    } else if (*s == 'r' && !strcmpstart(s, "router ")) {
+      *s_ptr = annotations ? annotations : s;
+      *is_extrainfo_out = 0;
+      return 0;
+    } else if (*s == 'e' && !strcmpstart(s, "extra-info ")) {
+      *s_ptr = annotations ? annotations : s;
+      *is_extrainfo_out = 1;
+      return 0;
+    }
+
+    if (!(s = memchr(s+1, '\n', eos-(s+1))))
+      break;
+    s = eat_whitespace_eos(s, eos);
+  }
+  return -1;
+}
+
+/** Given a string *<b>s</b> containing a concatenated sequence of router
+ * descriptors (or extra-info documents if <b>is_extrainfo</b> is set), parses
+ * them and stores the result in <b>dest</b>.  All routers are marked running
+ * and valid.  Advances *s to a point immediately following the last router
+ * entry.  Ignore any trailing router entries that are not complete.
+ *
+ * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each
+ * descriptor in the signed_descriptor_body field of each routerinfo_t.  If it
+ * isn't SAVED_NOWHERE, remember the offset of each descriptor.
+ *
+ * Returns 0 on success and -1 on failure.  Adds a digest to
+ * <b>invalid_digests_out</b> for every entry that was unparseable or
+ * invalid. (This may cause duplicate entries.)
+ */
+int
+router_parse_list_from_string(const char **s, const char *eos,
+                              smartlist_t *dest,
+                              saved_location_t saved_location,
+                              int want_extrainfo,
+                              int allow_annotations,
+                              const char *prepend_annotations,
+                              smartlist_t *invalid_digests_out)
+{
+  routerinfo_t *router;
+  extrainfo_t *extrainfo;
+  signed_descriptor_t *signed_desc = NULL;
+  void *elt;
+  const char *end, *start;
+  int have_extrainfo;
+
+  tor_assert(s);
+  tor_assert(*s);
+  tor_assert(dest);
+
+  start = *s;
+  if (!eos)
+    eos = *s + strlen(*s);
+
+  tor_assert(eos >= *s);
+
+  while (1) {
+    char raw_digest[DIGEST_LEN];
+    int have_raw_digest = 0;
+    int dl_again = 0;
+    if (find_start_of_next_router_or_extrainfo(s, eos, &have_extrainfo) < 0)
+      break;
+
+    end = tor_memstr(*s, eos-*s, "\nrouter-signature");
+    if (end)
+      end = tor_memstr(end, eos-end, "\n-----END SIGNATURE-----\n");
+    if (end)
+      end += strlen("\n-----END SIGNATURE-----\n");
+
+    if (!end)
+      break;
+
+    elt = NULL;
+
+    if (have_extrainfo && want_extrainfo) {
+      routerlist_t *rl = router_get_routerlist();
+      have_raw_digest = router_get_extrainfo_hash(*s, end-*s, raw_digest) == 0;
+      extrainfo = extrainfo_parse_entry_from_string(*s, end,
+                                       saved_location != SAVED_IN_CACHE,
+                                       rl->identity_map, &dl_again);
+      if (extrainfo) {
+        signed_desc = &extrainfo->cache_info;
+        elt = extrainfo;
+      }
+    } else if (!have_extrainfo && !want_extrainfo) {
+      have_raw_digest = router_get_router_hash(*s, end-*s, raw_digest) == 0;
+      router = router_parse_entry_from_string(*s, end,
+                                              saved_location != SAVED_IN_CACHE,
+                                              allow_annotations,
+                                              prepend_annotations, &dl_again);
+      if (router) {
+        log_debug(LD_DIR, "Read router '%s', purpose '%s'",
+                  router_describe(router),
+                  router_purpose_to_string(router->purpose));
+        signed_desc = &router->cache_info;
+        elt = router;
+      }
+    }
+    if (! elt && ! dl_again && have_raw_digest && invalid_digests_out) {
+      smartlist_add(invalid_digests_out, tor_memdup(raw_digest, DIGEST_LEN));
+    }
+    if (!elt) {
+      *s = end;
+      continue;
+    }
+    if (saved_location != SAVED_NOWHERE) {
+      tor_assert(signed_desc);
+      signed_desc->saved_location = saved_location;
+      signed_desc->saved_offset = *s - start;
+    }
+    *s = end;
+    smartlist_add(dest, elt);
+  }
+
+  return 0;
+}
+
+/* For debugging: define to count every descriptor digest we've seen so we
+ * know if we need to try harder to avoid duplicate verifies. */
+#undef COUNT_DISTINCT_DIGESTS
+
+#ifdef COUNT_DISTINCT_DIGESTS
+static digestmap_t *verified_digests = NULL;
+#endif
+
+/** Log the total count of the number of distinct router digests we've ever
+ * verified.  When compared to the number of times we've verified routerdesc
+ * signatures <i>in toto</i>, this will tell us if we're doing too much
+ * multiple-verification. */
+void
+dump_distinct_digest_count(int severity)
+{
+#ifdef COUNT_DISTINCT_DIGESTS
+  if (!verified_digests)
+    verified_digests = digestmap_new();
+  tor_log(severity, LD_GENERAL, "%d *distinct* router digests verified",
+      digestmap_size(verified_digests));
+#else /* !(defined(COUNT_DISTINCT_DIGESTS)) */
+  (void)severity; /* suppress "unused parameter" warning */
+#endif /* defined(COUNT_DISTINCT_DIGESTS) */
+}
+
+/** Try to find an IPv6 OR port in <b>list</b> of directory_token_t's
+ * with at least one argument (use GE(1) in setup). If found, store
+ * address and port number to <b>addr_out</b> and
+ * <b>port_out</b>. Return number of OR ports found. */
+static int
+find_single_ipv6_orport(const smartlist_t *list,
+                        tor_addr_t *addr_out,
+                        uint16_t *port_out)
+{
+  int ret = 0;
+  tor_assert(list != NULL);
+  tor_assert(addr_out != NULL);
+  tor_assert(port_out != NULL);
+
+  SMARTLIST_FOREACH_BEGIN(list, directory_token_t *, t) {
+    tor_addr_t a;
+    maskbits_t bits;
+    uint16_t port_min, port_max;
+    tor_assert(t->n_args >= 1);
+    /* XXXX Prop186 the full spec allows much more than this. */
+    if (tor_addr_parse_mask_ports(t->args[0], 0,
+                                  &a, &bits, &port_min,
+                                  &port_max) == AF_INET6 &&
+        bits == 128 &&
+        port_min == port_max) {
+      /* Okay, this is one we can understand. Use it and ignore
+         any potential more addresses in list. */
+      tor_addr_copy(addr_out, &a);
+      *port_out = port_min;
+      ret = 1;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(t);
+
+  return ret;
+}
+
+/** Helper function: reads a single router entry from *<b>s</b> ...
+ * *<b>end</b>.  Mallocs a new router and returns it if all goes well, else
+ * returns NULL.  If <b>cache_copy</b> is true, duplicate the contents of
+ * s through end into the signed_descriptor_body of the resulting
+ * routerinfo_t.
+ *
+ * If <b>end</b> is NULL, <b>s</b> must be properly NUL-terminated.
+ *
+ * If <b>allow_annotations</b>, it's okay to encounter annotations in <b>s</b>
+ * before the router; if it's false, reject the router if it's annotated.  If
+ * <b>prepend_annotations</b> is set, it should contain some annotations:
+ * append them to the front of the router before parsing it, and keep them
+ * around when caching the router.
+ *
+ * Only one of allow_annotations and prepend_annotations may be set.
+ *
+ * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1
+ * if it's okay to try to download a descriptor with this same digest again,
+ * and 0 if it isn't.  (It might not be okay to download it again if part of
+ * the part covered by the digest is invalid.)
+ */
+routerinfo_t *
+router_parse_entry_from_string(const char *s, const char *end,
+                               int cache_copy, int allow_annotations,
+                               const char *prepend_annotations,
+                               int *can_dl_again_out)
+{
+  routerinfo_t *router = NULL;
+  char digest[128];
+  smartlist_t *tokens = NULL, *exit_policy_tokens = NULL;
+  directory_token_t *tok;
+  struct in_addr in;
+  const char *start_of_annotations, *cp, *s_dup = s;
+  size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0;
+  int ok = 1;
+  memarea_t *area = NULL;
+  tor_cert_t *ntor_cc_cert = NULL;
+  /* Do not set this to '1' until we have parsed everything that we intend to
+   * parse that's covered by the hash. */
+  int can_dl_again = 0;
+
+  tor_assert(!allow_annotations || !prepend_annotations);
+
+  if (!end) {
+    end = s + strlen(s);
+  }
+
+  /* point 'end' to a point immediately after the final newline. */
+  while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n')
+    --end;
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (prepend_annotations) {
+    if (tokenize_string(area,prepend_annotations,NULL,tokens,
+                        routerdesc_token_table,TS_NOCHECK)) {
+      log_warn(LD_DIR, "Error tokenizing router descriptor (annotations).");
+      goto err;
+    }
+  }
+
+  start_of_annotations = s;
+  cp = tor_memstr(s, end-s, "\nrouter ");
+  if (!cp) {
+    if (end-s < 7 || strcmpstart(s, "router ")) {
+      log_warn(LD_DIR, "No router keyword found.");
+      goto err;
+    }
+  } else {
+    s = cp+1;
+  }
+
+  if (start_of_annotations != s) { /* We have annotations */
+    if (allow_annotations) {
+      if (tokenize_string(area,start_of_annotations,s,tokens,
+                          routerdesc_token_table,TS_NOCHECK)) {
+        log_warn(LD_DIR, "Error tokenizing router descriptor (annotations).");
+        goto err;
+      }
+    } else {
+      log_warn(LD_DIR, "Found unexpected annotations on router descriptor not "
+               "loaded from disk.  Dropping it.");
+      goto err;
+    }
+  }
+
+  if (router_get_router_hash(s, end - s, digest) < 0) {
+    log_warn(LD_DIR, "Couldn't compute router hash.");
+    goto err;
+  }
+  {
+    int flags = 0;
+    if (allow_annotations)
+      flags |= TS_ANNOTATIONS_OK;
+    if (prepend_annotations)
+      flags |= TS_ANNOTATIONS_OK|TS_NO_NEW_ANNOTATIONS;
+
+    if (tokenize_string(area,s,end,tokens,routerdesc_token_table, flags)) {
+      log_warn(LD_DIR, "Error tokenizing router descriptor.");
+      goto err;
+    }
+  }
+
+  if (smartlist_len(tokens) < 2) {
+    log_warn(LD_DIR, "Impossibly short router descriptor.");
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_ROUTER);
+  const int router_token_pos = smartlist_pos(tokens, tok);
+  tor_assert(tok->n_args >= 5);
+
+  router = tor_malloc_zero(sizeof(routerinfo_t));
+  router->cert_expiration_time = TIME_MAX;
+  router->cache_info.routerlist_index = -1;
+  router->cache_info.annotations_len = s-start_of_annotations + prepend_len;
+  router->cache_info.signed_descriptor_len = end-s;
+  if (cache_copy) {
+    size_t len = router->cache_info.signed_descriptor_len +
+                 router->cache_info.annotations_len;
+    char *signed_body =
+      router->cache_info.signed_descriptor_body = tor_malloc(len+1);
+    if (prepend_annotations) {
+      memcpy(signed_body, prepend_annotations, prepend_len);
+      signed_body += prepend_len;
+    }
+    /* This assertion will always succeed.
+     * len == signed_desc_len + annotations_len
+     *     == end-s + s-start_of_annotations + prepend_len
+     *     == end-start_of_annotations + prepend_len
+     * We already wrote prepend_len bytes into the buffer; now we're
+     * writing end-start_of_annotations -NM. */
+    tor_assert(signed_body+(end-start_of_annotations) ==
+               router->cache_info.signed_descriptor_body+len);
+    memcpy(signed_body, start_of_annotations, end-start_of_annotations);
+    router->cache_info.signed_descriptor_body[len] = '\0';
+    tor_assert(strlen(router->cache_info.signed_descriptor_body) == len);
+  }
+  memcpy(router->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
+
+  router->nickname = tor_strdup(tok->args[0]);
+  if (!is_legal_nickname(router->nickname)) {
+    log_warn(LD_DIR,"Router nickname is invalid");
+    goto err;
+  }
+  if (!tor_inet_aton(tok->args[1], &in)) {
+    log_warn(LD_DIR,"Router address is not an IP address.");
+    goto err;
+  }
+  router->addr = ntohl(in.s_addr);
+
+  router->or_port =
+    (uint16_t) tor_parse_long(tok->args[2],10,0,65535,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR,"Invalid OR port %s", escaped(tok->args[2]));
+    goto err;
+  }
+  router->dir_port =
+    (uint16_t) tor_parse_long(tok->args[4],10,0,65535,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR,"Invalid dir port %s", escaped(tok->args[4]));
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_BANDWIDTH);
+  tor_assert(tok->n_args >= 3);
+  router->bandwidthrate = (int)
+    tor_parse_long(tok->args[0],10,1,INT_MAX,&ok,NULL);
+
+  if (!ok) {
+    log_warn(LD_DIR, "bandwidthrate %s unreadable or 0. Failing.",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  router->bandwidthburst =
+    (int) tor_parse_long(tok->args[1],10,0,INT_MAX,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR, "Invalid bandwidthburst %s", escaped(tok->args[1]));
+    goto err;
+  }
+  router->bandwidthcapacity = (int)
+    tor_parse_long(tok->args[2],10,0,INT_MAX,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR, "Invalid bandwidthcapacity %s", escaped(tok->args[1]));
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, A_PURPOSE))) {
+    tor_assert(tok->n_args);
+    router->purpose = router_purpose_from_string(tok->args[0]);
+  } else {
+    router->purpose = ROUTER_PURPOSE_GENERAL;
+  }
+  router->cache_info.send_unencrypted =
+    (router->purpose == ROUTER_PURPOSE_GENERAL) ? 1 : 0;
+
+  if ((tok = find_opt_by_keyword(tokens, K_UPTIME))) {
+    tor_assert(tok->n_args >= 1);
+    router->uptime = tor_parse_long(tok->args[0],10,0,LONG_MAX,&ok,NULL);
+    if (!ok) {
+      log_warn(LD_DIR, "Invalid uptime %s", escaped(tok->args[0]));
+      goto err;
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_HIBERNATING))) {
+    tor_assert(tok->n_args >= 1);
+    router->is_hibernating
+      = (tor_parse_long(tok->args[0],10,0,LONG_MAX,NULL,NULL) != 0);
+  }
+
+  tok = find_by_keyword(tokens, K_PUBLISHED);
+  tor_assert(tok->n_args == 1);
+  if (parse_iso_time(tok->args[0], &router->cache_info.published_on) < 0)
+    goto err;
+
+  tok = find_by_keyword(tokens, K_ONION_KEY);
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_DIR,
+             "Relay's onion key had invalid exponent.");
+    goto err;
+  }
+  router->onion_pkey = tok->key;
+  tok->key = NULL; /* Prevent free */
+
+  if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+    curve25519_public_key_t k;
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus ntor-onion-key in routerinfo");
+      goto err;
+    }
+    router->onion_curve25519_pkey =
+      tor_memdup(&k, sizeof(curve25519_public_key_t));
+  }
+
+  tok = find_by_keyword(tokens, K_SIGNING_KEY);
+  router->identity_pkey = tok->key;
+  tok->key = NULL; /* Prevent free */
+  if (crypto_pk_get_digest(router->identity_pkey,
+                           router->cache_info.identity_digest)) {
+    log_warn(LD_DIR, "Couldn't calculate key digest"); goto err;
+  }
+
+  {
+    directory_token_t *ed_sig_tok, *ed_cert_tok, *cc_tap_tok, *cc_ntor_tok,
+      *master_key_tok;
+    ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519);
+    ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519);
+    master_key_tok = find_opt_by_keyword(tokens, K_MASTER_KEY_ED25519);
+    cc_tap_tok = find_opt_by_keyword(tokens, K_ONION_KEY_CROSSCERT);
+    cc_ntor_tok = find_opt_by_keyword(tokens, K_NTOR_ONION_KEY_CROSSCERT);
+    int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok +
+      !!cc_tap_tok + !!cc_ntor_tok;
+    if ((n_ed_toks != 0 && n_ed_toks != 4) ||
+        (n_ed_toks == 4 && !router->onion_curve25519_pkey)) {
+      log_warn(LD_DIR, "Router descriptor with only partial ed25519/"
+               "cross-certification support");
+      goto err;
+    }
+    if (master_key_tok && !ed_sig_tok) {
+      log_warn(LD_DIR, "Router descriptor has ed25519 master key but no "
+               "certificate");
+      goto err;
+    }
+    if (ed_sig_tok) {
+      tor_assert(ed_cert_tok && cc_tap_tok && cc_ntor_tok);
+      const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok);
+      if (ed_cert_token_pos == -1 || router_token_pos == -1 ||
+          (ed_cert_token_pos != router_token_pos + 1 &&
+           ed_cert_token_pos != router_token_pos - 1)) {
+        log_warn(LD_DIR, "Ed25519 certificate in wrong position");
+        goto err;
+      }
+      if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) {
+        log_warn(LD_DIR, "Ed25519 signature in wrong position");
+        goto err;
+      }
+      if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) {
+        log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor");
+        goto err;
+      }
+      if (strcmp(cc_ntor_tok->object_type, "ED25519 CERT")) {
+        log_warn(LD_DIR, "Wrong object type on ntor-onion-key-crosscert "
+                 "in decriptor");
+        goto err;
+      }
+      if (strcmp(cc_tap_tok->object_type, "CROSSCERT")) {
+        log_warn(LD_DIR, "Wrong object type on onion-key-crosscert "
+                 "in decriptor");
+        goto err;
+      }
+      if (strcmp(cc_ntor_tok->args[0], "0") &&
+          strcmp(cc_ntor_tok->args[0], "1")) {
+        log_warn(LD_DIR, "Bad sign bit on ntor-onion-key-crosscert");
+        goto err;
+      }
+      int ntor_cc_sign_bit = !strcmp(cc_ntor_tok->args[0], "1");
+
+      uint8_t d256[DIGEST256_LEN];
+      const char *signed_start, *signed_end;
+      tor_cert_t *cert = tor_cert_parse(
+                       (const uint8_t*)ed_cert_tok->object_body,
+                       ed_cert_tok->object_size);
+      if (! cert) {
+        log_warn(LD_DIR, "Couldn't parse ed25519 cert");
+        goto err;
+      }
+      /* makes sure it gets freed. */
+      router->cache_info.signing_key_cert = cert;
+
+      if (cert->cert_type != CERT_TYPE_ID_SIGNING ||
+          ! cert->signing_key_included) {
+        log_warn(LD_DIR, "Invalid form for ed25519 cert");
+        goto err;
+      }
+
+      if (master_key_tok) {
+        /* This token is optional, but if it's present, it must match
+         * the signature in the signing cert, or supplant it. */
+        tor_assert(master_key_tok->n_args >= 1);
+        ed25519_public_key_t pkey;
+        if (ed25519_public_from_base64(&pkey, master_key_tok->args[0])<0) {
+          log_warn(LD_DIR, "Can't parse ed25519 master key");
+          goto err;
+        }
+
+        if (fast_memneq(&cert->signing_key.pubkey,
+                        pkey.pubkey, ED25519_PUBKEY_LEN)) {
+          log_warn(LD_DIR, "Ed25519 master key does not match "
+                   "key in certificate");
+          goto err;
+        }
+      }
+      ntor_cc_cert = tor_cert_parse((const uint8_t*)cc_ntor_tok->object_body,
+                                    cc_ntor_tok->object_size);
+      if (!ntor_cc_cert) {
+        log_warn(LD_DIR, "Couldn't parse ntor-onion-key-crosscert cert");
+        goto err;
+      }
+      if (ntor_cc_cert->cert_type != CERT_TYPE_ONION_ID ||
+          ! ed25519_pubkey_eq(&ntor_cc_cert->signed_key, &cert->signing_key)) {
+        log_warn(LD_DIR, "Invalid contents for ntor-onion-key-crosscert cert");
+        goto err;
+      }
+
+      ed25519_public_key_t ntor_cc_pk;
+      if (ed25519_public_key_from_curve25519_public_key(&ntor_cc_pk,
+                                            router->onion_curve25519_pkey,
+                                            ntor_cc_sign_bit)<0) {
+        log_warn(LD_DIR, "Error converting onion key to ed25519");
+        goto err;
+      }
+
+      if (router_get_hash_impl_helper(s, end-s, "router ",
+                                      "\nrouter-sig-ed25519",
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
+        log_warn(LD_DIR, "Can't find ed25519-signed portion of descriptor");
+        goto err;
+      }
+      crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+      crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX,
+        strlen(ED_DESC_SIGNATURE_PREFIX));
+      crypto_digest_add_bytes(d, signed_start, signed_end-signed_start);
+      crypto_digest_get_digest(d, (char*)d256, sizeof(d256));
+      crypto_digest_free(d);
+
+      ed25519_checkable_t check[3];
+      int check_ok[3];
+      time_t expires = TIME_MAX;
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, &expires) < 0) {
+        log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
+        goto err;
+      }
+      if (tor_cert_get_checkable_sig(&check[1],
+                               ntor_cc_cert, &ntor_cc_pk, &expires) < 0) {
+        log_err(LD_BUG, "Couldn't create 'checkable' for ntor_cc_cert.");
+        goto err;
+      }
+
+      if (ed25519_signature_from_base64(&check[2].signature,
+                                        ed_sig_tok->args[0])<0) {
+        log_warn(LD_DIR, "Couldn't decode ed25519 signature");
+        goto err;
+      }
+      check[2].pubkey = &cert->signed_key;
+      check[2].msg = d256;
+      check[2].len = DIGEST256_LEN;
+
+      if (ed25519_checksig_batch(check_ok, check, 3) < 0) {
+        log_warn(LD_DIR, "Incorrect ed25519 signature(s)");
+        goto err;
+      }
+
+      if (check_tap_onion_key_crosscert(
+                      (const uint8_t*)cc_tap_tok->object_body,
+                      (int)cc_tap_tok->object_size,
+                      router->onion_pkey,
+                      &cert->signing_key,
+                      (const uint8_t*)router->cache_info.identity_digest)<0) {
+        log_warn(LD_DIR, "Incorrect TAP cross-verification");
+        goto err;
+      }
+
+      /* We check this before adding it to the routerlist. */
+      router->cert_expiration_time = expires;
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_FINGERPRINT))) {
+    /* If there's a fingerprint line, it must match the identity digest. */
+    char d[DIGEST_LEN];
+    tor_assert(tok->n_args == 1);
+    tor_strstrip(tok->args[0], " ");
+    if (base16_decode(d, DIGEST_LEN,
+                      tok->args[0], strlen(tok->args[0])) != DIGEST_LEN) {
+      log_warn(LD_DIR, "Couldn't decode router fingerprint %s",
+               escaped(tok->args[0]));
+      goto err;
+    }
+    if (tor_memneq(d,router->cache_info.identity_digest, DIGEST_LEN)) {
+      log_warn(LD_DIR, "Fingerprint '%s' does not match identity digest.",
+               tok->args[0]);
+      goto err;
+    }
+  }
+
+  {
+    const char *version = NULL, *protocols = NULL;
+    if ((tok = find_opt_by_keyword(tokens, K_PLATFORM))) {
+      router->platform = tor_strdup(tok->args[0]);
+      version = tok->args[0];
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
+      router->protocol_list = tor_strdup(tok->args[0]);
+      protocols = tok->args[0];
+    }
+
+    summarize_protover_flags(&router->pv, protocols, version);
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_CONTACT))) {
+    router->contact_info = tor_strdup(tok->args[0]);
+  }
+
+  if (find_opt_by_keyword(tokens, K_REJECT6) ||
+      find_opt_by_keyword(tokens, K_ACCEPT6)) {
+    log_warn(LD_DIR, "Rejecting router with reject6/accept6 line: they crash "
+             "older Tors.");
+    goto err;
+  }
+  {
+    smartlist_t *or_addresses = find_all_by_keyword(tokens, K_OR_ADDRESS);
+    if (or_addresses) {
+      find_single_ipv6_orport(or_addresses, &router->ipv6_addr,
+                              &router->ipv6_orport);
+      smartlist_free(or_addresses);
+    }
+  }
+  exit_policy_tokens = find_all_exitpolicy(tokens);
+  if (!smartlist_len(exit_policy_tokens)) {
+    log_warn(LD_DIR, "No exit policy tokens in descriptor.");
+    goto err;
+  }
+  SMARTLIST_FOREACH(exit_policy_tokens, directory_token_t *, t,
+                    if (router_add_exit_policy(router,t)<0) {
+                      log_warn(LD_DIR,"Error in exit policy");
+                      goto err;
+                    });
+  policy_expand_private(&router->exit_policy);
+
+  if ((tok = find_opt_by_keyword(tokens, K_IPV6_POLICY)) && tok->n_args) {
+    router->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+    if (! router->ipv6_exit_policy) {
+      log_warn(LD_DIR , "Error in ipv6-policy %s", escaped(tok->args[0]));
+      goto err;
+    }
+  }
+
+  if (policy_is_reject_star(router->exit_policy, AF_INET, 1) &&
+      (!router->ipv6_exit_policy ||
+       short_policy_is_reject_star(router->ipv6_exit_policy)))
+    router->policy_is_reject_star = 1;
+
+  if ((tok = find_opt_by_keyword(tokens, K_FAMILY)) && tok->n_args) {
+    int i;
+    router->declared_family = smartlist_new();
+    for (i=0;i<tok->n_args;++i) {
+      if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
+        log_warn(LD_DIR, "Illegal nickname %s in family line",
+                 escaped(tok->args[i]));
+        goto err;
+      }
+      smartlist_add_strdup(router->declared_family, tok->args[i]);
+    }
+  }
+
+  if (find_opt_by_keyword(tokens, K_CACHES_EXTRA_INFO))
+    router->caches_extra_info = 1;
+
+  if (find_opt_by_keyword(tokens, K_ALLOW_SINGLE_HOP_EXITS))
+    router->allow_single_hop_exits = 1;
+
+  if ((tok = find_opt_by_keyword(tokens, K_EXTRA_INFO_DIGEST))) {
+    tor_assert(tok->n_args >= 1);
+    if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
+      if (base16_decode(router->cache_info.extra_info_digest, DIGEST_LEN,
+                        tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN) {
+          log_warn(LD_DIR,"Invalid extra info digest");
+      }
+    } else {
+      log_warn(LD_DIR, "Invalid extra info digest %s", escaped(tok->args[0]));
+    }
+
+    if (tok->n_args >= 2) {
+      if (digest256_from_base64(router->cache_info.extra_info_digest256,
+                                tok->args[1]) < 0) {
+        log_warn(LD_DIR, "Invalid extra info digest256 %s",
+                 escaped(tok->args[1]));
+      }
+    }
+  }
+
+  if (find_opt_by_keyword(tokens, K_HIDDEN_SERVICE_DIR)) {
+    router->wants_to_be_hs_dir = 1;
+  }
+
+  /* This router accepts tunnelled directory requests via begindir if it has
+   * an open dirport or it included "tunnelled-dir-server". */
+  if (find_opt_by_keyword(tokens, K_DIR_TUNNELLED) || router->dir_port > 0) {
+    router->supports_tunnelled_dir_requests = 1;
+  }
+
+  tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
+#ifdef COUNT_DISTINCT_DIGESTS
+  if (!verified_digests)
+    verified_digests = digestmap_new();
+  digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1);
+#endif
+
+  if (!router->or_port) {
+    log_warn(LD_DIR,"or_port unreadable or 0. Failing.");
+    goto err;
+  }
+
+  /* We've checked everything that's covered by the hash. */
+  can_dl_again = 1;
+  if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0,
+                            "router descriptor") < 0)
+    goto err;
+
+  if (!router->platform) {
+    router->platform = tor_strdup("<unknown>");
+  }
+  goto done;
+
+ err:
+  dump_desc(s_dup, "router descriptor");
+  routerinfo_free(router);
+  router = NULL;
+ done:
+  tor_cert_free(ntor_cc_cert);
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  smartlist_free(exit_policy_tokens);
+  if (area) {
+    DUMP_AREA(area, "routerinfo");
+    memarea_drop_all(area);
+  }
+  if (can_dl_again_out)
+    *can_dl_again_out = can_dl_again;
+  return router;
+}
+
+/** Parse a single extrainfo entry from the string <b>s</b>, ending at
+ * <b>end</b>.  (If <b>end</b> is NULL, parse up to the end of <b>s</b>.)  If
+ * <b>cache_copy</b> is true, make a copy of the extra-info document in the
+ * cache_info fields of the result.  If <b>routermap</b> is provided, use it
+ * as a map from router identity to routerinfo_t when looking up signing keys.
+ *
+ * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1
+ * if it's okay to try to download an extrainfo with this same digest again,
+ * and 0 if it isn't.  (It might not be okay to download it again if part of
+ * the part covered by the digest is invalid.)
+ */
+extrainfo_t *
+extrainfo_parse_entry_from_string(const char *s, const char *end,
+                            int cache_copy, struct digest_ri_map_t *routermap,
+                            int *can_dl_again_out)
+{
+  extrainfo_t *extrainfo = NULL;
+  char digest[128];
+  smartlist_t *tokens = NULL;
+  directory_token_t *tok;
+  crypto_pk_t *key = NULL;
+  routerinfo_t *router = NULL;
+  memarea_t *area = NULL;
+  const char *s_dup = s;
+  /* Do not set this to '1' until we have parsed everything that we intend to
+   * parse that's covered by the hash. */
+  int can_dl_again = 0;
+
+  if (BUG(s == NULL))
+    return NULL;
+
+  if (!end) {
+    end = s + strlen(s);
+  }
+
+  /* point 'end' to a point immediately after the final newline. */
+  while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n')
+    --end;
+
+  if (router_get_extrainfo_hash(s, end-s, digest) < 0) {
+    log_warn(LD_DIR, "Couldn't compute router hash.");
+    goto err;
+  }
+  tokens = smartlist_new();
+  area = memarea_new();
+  if (tokenize_string(area,s,end,tokens,extrainfo_token_table,0)) {
+    log_warn(LD_DIR, "Error tokenizing extra-info document.");
+    goto err;
+  }
+
+  if (smartlist_len(tokens) < 2) {
+    log_warn(LD_DIR, "Impossibly short extra-info document.");
+    goto err;
+  }
+
+  /* XXXX Accept this in position 1 too, and ed identity in position 0. */
+  tok = smartlist_get(tokens,0);
+  if (tok->tp != K_EXTRA_INFO) {
+    log_warn(LD_DIR,"Entry does not start with \"extra-info\"");
+    goto err;
+  }
+
+  extrainfo = tor_malloc_zero(sizeof(extrainfo_t));
+  extrainfo->cache_info.is_extrainfo = 1;
+  if (cache_copy)
+    extrainfo->cache_info.signed_descriptor_body = tor_memdup_nulterm(s,end-s);
+  extrainfo->cache_info.signed_descriptor_len = end-s;
+  memcpy(extrainfo->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
+  crypto_digest256((char*)extrainfo->digest256, s, end-s, DIGEST_SHA256);
+
+  tor_assert(tok->n_args >= 2);
+  if (!is_legal_nickname(tok->args[0])) {
+    log_warn(LD_DIR,"Bad nickname %s on \"extra-info\"",escaped(tok->args[0]));
+    goto err;
+  }
+  strlcpy(extrainfo->nickname, tok->args[0], sizeof(extrainfo->nickname));
+  if (strlen(tok->args[1]) != HEX_DIGEST_LEN ||
+      base16_decode(extrainfo->cache_info.identity_digest, DIGEST_LEN,
+                    tok->args[1], HEX_DIGEST_LEN) != DIGEST_LEN) {
+    log_warn(LD_DIR,"Invalid fingerprint %s on \"extra-info\"",
+             escaped(tok->args[1]));
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_PUBLISHED);
+  if (parse_iso_time(tok->args[0], &extrainfo->cache_info.published_on)) {
+    log_warn(LD_DIR,"Invalid published time %s on \"extra-info\"",
+             escaped(tok->args[0]));
+    goto err;
+  }
+
+  {
+    directory_token_t *ed_sig_tok, *ed_cert_tok;
+    ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519);
+    ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519);
+    int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok;
+    if (n_ed_toks != 0 && n_ed_toks != 2) {
+      log_warn(LD_DIR, "Router descriptor with only partial ed25519/"
+               "cross-certification support");
+      goto err;
+    }
+    if (ed_sig_tok) {
+      tor_assert(ed_cert_tok);
+      const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok);
+      if (ed_cert_token_pos != 1) {
+        /* Accept this in position 0 XXXX */
+        log_warn(LD_DIR, "Ed25519 certificate in wrong position");
+        goto err;
+      }
+      if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) {
+        log_warn(LD_DIR, "Ed25519 signature in wrong position");
+        goto err;
+      }
+      if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) {
+        log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor");
+        goto err;
+      }
+
+      uint8_t d256[DIGEST256_LEN];
+      const char *signed_start, *signed_end;
+      tor_cert_t *cert = tor_cert_parse(
+                       (const uint8_t*)ed_cert_tok->object_body,
+                       ed_cert_tok->object_size);
+      if (! cert) {
+        log_warn(LD_DIR, "Couldn't parse ed25519 cert");
+        goto err;
+      }
+      /* makes sure it gets freed. */
+      extrainfo->cache_info.signing_key_cert = cert;
+
+      if (cert->cert_type != CERT_TYPE_ID_SIGNING ||
+          ! cert->signing_key_included) {
+        log_warn(LD_DIR, "Invalid form for ed25519 cert");
+        goto err;
+      }
+
+      if (router_get_hash_impl_helper(s, end-s, "extra-info ",
+                                      "\nrouter-sig-ed25519",
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
+        log_warn(LD_DIR, "Can't find ed25519-signed portion of extrainfo");
+        goto err;
+      }
+      crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+      crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX,
+        strlen(ED_DESC_SIGNATURE_PREFIX));
+      crypto_digest_add_bytes(d, signed_start, signed_end-signed_start);
+      crypto_digest_get_digest(d, (char*)d256, sizeof(d256));
+      crypto_digest_free(d);
+
+      ed25519_checkable_t check[2];
+      int check_ok[2];
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, NULL) < 0) {
+        log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
+        goto err;
+      }
+
+      if (ed25519_signature_from_base64(&check[1].signature,
+                                        ed_sig_tok->args[0])<0) {
+        log_warn(LD_DIR, "Couldn't decode ed25519 signature");
+        goto err;
+      }
+      check[1].pubkey = &cert->signed_key;
+      check[1].msg = d256;
+      check[1].len = DIGEST256_LEN;
+
+      if (ed25519_checksig_batch(check_ok, check, 2) < 0) {
+        log_warn(LD_DIR, "Incorrect ed25519 signature(s)");
+        goto err;
+      }
+      /* We don't check the certificate expiration time: checking that it
+       * matches the cert in the router descriptor is adequate. */
+    }
+  }
+
+  /* We've checked everything that's covered by the hash. */
+  can_dl_again = 1;
+
+  if (routermap &&
+      (router = digestmap_get((digestmap_t*)routermap,
+                              extrainfo->cache_info.identity_digest))) {
+    key = router->identity_pkey;
+  }
+
+  tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
+  if (strcmp(tok->object_type, "SIGNATURE") ||
+      tok->object_size < 128 || tok->object_size > 512) {
+    log_warn(LD_DIR, "Bad object type or length on extra-info signature");
+    goto err;
+  }
+
+  if (key) {
+    if (check_signature_token(digest, DIGEST_LEN, tok, key, 0,
+                              "extra-info") < 0)
+      goto err;
+
+    if (router)
+      extrainfo->cache_info.send_unencrypted =
+        router->cache_info.send_unencrypted;
+  } else {
+    extrainfo->pending_sig = tor_memdup(tok->object_body,
+                                        tok->object_size);
+    extrainfo->pending_sig_len = tok->object_size;
+  }
+
+  goto done;
+ err:
+  dump_desc(s_dup, "extra-info descriptor");
+  extrainfo_free(extrainfo);
+  extrainfo = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    DUMP_AREA(area, "extrainfo");
+    memarea_drop_all(area);
+  }
+  if (can_dl_again_out)
+    *can_dl_again_out = can_dl_again;
+  return extrainfo;
+}
+
+/** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to
+ * the first character after the certificate. */
+authority_cert_t *
+authority_cert_parse_from_string(const char *s, const char **end_of_string)
+{
+  /** Reject any certificate at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_CERT_SIZE (128*1024)
+
+  authority_cert_t *cert = NULL, *old_cert;
+  smartlist_t *tokens = NULL;
+  char digest[DIGEST_LEN];
+  directory_token_t *tok;
+  char fp_declared[DIGEST_LEN];
+  char *eos;
+  size_t len;
+  int found;
+  memarea_t *area = NULL;
+  const char *s_dup = s;
+
+  s = eat_whitespace(s);
+  eos = strstr(s, "\ndir-key-certification");
+  if (! eos) {
+    log_warn(LD_DIR, "No signature found on key certificate");
+    return NULL;
+  }
+  eos = strstr(eos, "\n-----END SIGNATURE-----\n");
+  if (! eos) {
+    log_warn(LD_DIR, "No end-of-signature found on key certificate");
+    return NULL;
+  }
+  eos = strchr(eos+2, '\n');
+  tor_assert(eos);
+  ++eos;
+  len = eos - s;
+
+  if (len > MAX_CERT_SIZE) {
+    log_warn(LD_DIR, "Certificate is far too big (at %lu bytes long); "
+             "rejecting", (unsigned long)len);
+    return NULL;
+  }
+
+  tokens = smartlist_new();
+  area = memarea_new();
+  if (tokenize_string(area,s, eos, tokens, dir_key_certificate_table, 0) < 0) {
+    log_warn(LD_DIR, "Error tokenizing key certificate");
+    goto err;
+  }
+  if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version",
+                           "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
+    goto err;
+  tok = smartlist_get(tokens, 0);
+  if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) {
+    log_warn(LD_DIR,
+             "Key certificate does not begin with a recognized version (3).");
+    goto err;
+  }
+
+  cert = tor_malloc_zero(sizeof(authority_cert_t));
+  memcpy(cert->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
+
+  tok = find_by_keyword(tokens, K_DIR_SIGNING_KEY);
+  tor_assert(tok->key);
+  cert->signing_key = tok->key;
+  tok->key = NULL;
+  if (crypto_pk_get_digest(cert->signing_key, cert->signing_key_digest))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_DIR_IDENTITY_KEY);
+  tor_assert(tok->key);
+  cert->identity_key = tok->key;
+  tok->key = NULL;
+
+  tok = find_by_keyword(tokens, K_FINGERPRINT);
+  tor_assert(tok->n_args);
+  if (base16_decode(fp_declared, DIGEST_LEN, tok->args[0],
+                    strlen(tok->args[0])) != DIGEST_LEN) {
+    log_warn(LD_DIR, "Couldn't decode key certificate fingerprint %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+
+  if (crypto_pk_get_digest(cert->identity_key,
+                           cert->cache_info.identity_digest))
+    goto err;
+
+  if (tor_memneq(cert->cache_info.identity_digest, fp_declared, DIGEST_LEN)) {
+    log_warn(LD_DIR, "Digest of certificate key didn't match declared "
+             "fingerprint");
+    goto err;
+  }
+
+  tok = find_opt_by_keyword(tokens, K_DIR_ADDRESS);
+  if (tok) {
+    struct in_addr in;
+    char *address = NULL;
+    tor_assert(tok->n_args);
+    /* XXX++ use some tor_addr parse function below instead. -RD */
+    if (tor_addr_port_split(LOG_WARN, tok->args[0], &address,
+                            &cert->dir_port) < 0 ||
+        tor_inet_aton(address, &in) == 0) {
+      log_warn(LD_DIR, "Couldn't parse dir-address in certificate");
+      tor_free(address);
+      goto err;
+    }
+    cert->addr = ntohl(in.s_addr);
+    tor_free(address);
+  }
+
+  tok = find_by_keyword(tokens, K_DIR_KEY_PUBLISHED);
+  if (parse_iso_time(tok->args[0], &cert->cache_info.published_on) < 0) {
+     goto err;
+  }
+  tok = find_by_keyword(tokens, K_DIR_KEY_EXPIRES);
+  if (parse_iso_time(tok->args[0], &cert->expires) < 0) {
+     goto err;
+  }
+
+  tok = smartlist_get(tokens, smartlist_len(tokens)-1);
+  if (tok->tp != K_DIR_KEY_CERTIFICATION) {
+    log_warn(LD_DIR, "Certificate didn't end with dir-key-certification.");
+    goto err;
+  }
+
+  /* If we already have this cert, don't bother checking the signature. */
+  old_cert = authority_cert_get_by_digests(
+                                     cert->cache_info.identity_digest,
+                                     cert->signing_key_digest);
+  found = 0;
+  if (old_cert) {
+    /* XXXX We could just compare signed_descriptor_digest, but that wouldn't
+     * buy us much. */
+    if (old_cert->cache_info.signed_descriptor_len == len &&
+        old_cert->cache_info.signed_descriptor_body &&
+        tor_memeq(s, old_cert->cache_info.signed_descriptor_body, len)) {
+      log_debug(LD_DIR, "We already checked the signature on this "
+                "certificate; no need to do so again.");
+      found = 1;
+    }
+  }
+  if (!found) {
+    if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0,
+                              "key certificate")) {
+      goto err;
+    }
+
+    tok = find_by_keyword(tokens, K_DIR_KEY_CROSSCERT);
+    if (check_signature_token(cert->cache_info.identity_digest,
+                              DIGEST_LEN,
+                              tok,
+                              cert->signing_key,
+                              CST_NO_CHECK_OBJTYPE,
+                              "key cross-certification")) {
+      goto err;
+    }
+  }
+
+  cert->cache_info.signed_descriptor_len = len;
+  cert->cache_info.signed_descriptor_body = tor_malloc(len+1);
+  memcpy(cert->cache_info.signed_descriptor_body, s, len);
+  cert->cache_info.signed_descriptor_body[len] = 0;
+  cert->cache_info.saved_location = SAVED_NOWHERE;
+
+  if (end_of_string) {
+    *end_of_string = eat_whitespace(eos);
+  }
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    DUMP_AREA(area, "authority cert");
+    memarea_drop_all(area);
+  }
+  return cert;
+ err:
+  dump_desc(s_dup, "authority cert");
+  authority_cert_free(cert);
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    DUMP_AREA(area, "authority cert");
+    memarea_drop_all(area);
+  }
+  return NULL;
+}
+
+/** Helper: given a string <b>s</b>, return the start of the next router-status
+ * object (starting with "r " at the start of a line).  If none is found,
+ * return the start of the directory footer, or the next directory signature.
+ * If none is found, return the end of the string. */
+static inline const char *
+find_start_of_next_routerstatus(const char *s)
+{
+  const char *eos, *footer, *sig;
+  if ((eos = strstr(s, "\nr ")))
+    ++eos;
+  else
+    eos = s + strlen(s);
+
+  footer = tor_memstr(s, eos-s, "\ndirectory-footer");
+  sig = tor_memstr(s, eos-s, "\ndirectory-signature");
+
+  if (footer && sig)
+    return MIN(footer, sig) + 1;
+  else if (footer)
+    return footer+1;
+  else if (sig)
+    return sig+1;
+  else
+    return eos;
+}
+
+/** Parse the GuardFraction string from a consensus or vote.
+ *
+ *  If <b>vote</b> or <b>vote_rs</b> are set the document getting
+ *  parsed is a vote routerstatus. Otherwise it's a consensus. This is
+ *  the same semantic as in routerstatus_parse_entry_from_string(). */
+STATIC int
+routerstatus_parse_guardfraction(const char *guardfraction_str,
+                                 networkstatus_t *vote,
+                                 vote_routerstatus_t *vote_rs,
+                                 routerstatus_t *rs)
+{
+  int ok;
+  const char *end_of_header = NULL;
+  int is_consensus = !vote_rs;
+  uint32_t guardfraction;
+
+  tor_assert(bool_eq(vote, vote_rs));
+
+  /* If this info comes from a consensus, but we should't apply
+     guardfraction, just exit. */
+  if (is_consensus && !should_apply_guardfraction(NULL)) {
+    return 0;
+  }
+
+  end_of_header = strchr(guardfraction_str, '=');
+  if (!end_of_header) {
+    return -1;
+  }
+
+  guardfraction = (uint32_t)tor_parse_ulong(end_of_header+1,
+                                            10, 0, 100, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_DIR, "Invalid GuardFraction %s", escaped(guardfraction_str));
+    return -1;
+  }
+
+  log_debug(LD_GENERAL, "[*] Parsed %s guardfraction '%s' for '%s'.",
+            is_consensus ? "consensus" : "vote",
+            guardfraction_str, rs->nickname);
+
+  if (!is_consensus) { /* We are parsing a vote */
+    vote_rs->status.guardfraction_percentage = guardfraction;
+    vote_rs->status.has_guardfraction = 1;
+  } else {
+    /* We are parsing a consensus. Only apply guardfraction to guards. */
+    if (rs->is_possible_guard) {
+      rs->guardfraction_percentage = guardfraction;
+      rs->has_guardfraction = 1;
+    } else {
+      log_warn(LD_BUG, "Got GuardFraction for non-guard %s. "
+               "This is not supposed to happen. Not applying. ", rs->nickname);
+    }
+  }
+
+  return 0;
+}
+
+/** Summarize the protocols listed in <b>protocols</b> into <b>out</b>,
+ * falling back or correcting them based on <b>version</b> as appropriate.
+ */
+STATIC void
+summarize_protover_flags(protover_summary_flags_t *out,
+                         const char *protocols,
+                         const char *version)
+{
+  tor_assert(out);
+  memset(out, 0, sizeof(*out));
+  if (protocols) {
+    out->protocols_known = 1;
+    out->supports_extend2_cells =
+      protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
+    out->supports_ed25519_link_handshake_compat =
+      protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
+    out->supports_ed25519_link_handshake_any =
+      protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
+    out->supports_ed25519_hs_intro =
+      protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
+    out->supports_v3_hsdir =
+      protocol_list_supports_protocol(protocols, PRT_HSDIR,
+                                      PROTOVER_HSDIR_V3);
+    out->supports_v3_rendezvous_point =
+      protocol_list_supports_protocol(protocols, PRT_HSREND,
+                                      PROTOVER_HS_RENDEZVOUS_POINT_V3);
+  }
+  if (version && !strcmpstart(version, "Tor ")) {
+    if (!out->protocols_known) {
+      /* The version is a "Tor" version, and where there is no
+       * list of protocol versions that we should be looking at instead. */
+
+      out->supports_extend2_cells =
+        tor_version_as_new_as(version, "0.2.4.8-alpha");
+      out->protocols_known = 1;
+    } else {
+      /* Bug #22447 forces us to filter on this version. */
+      if (!tor_version_as_new_as(version, "0.3.0.8")) {
+        out->supports_v3_hsdir = 0;
+      }
+    }
+  }
+}
+
+/** Given a string at *<b>s</b>, containing a routerstatus object, and an
+ * empty smartlist at <b>tokens</b>, parse and return the first router status
+ * object in the string, and advance *<b>s</b> to just after the end of the
+ * router status.  Return NULL and advance *<b>s</b> on error.
+ *
+ * If <b>vote</b> and <b>vote_rs</b> are provided, don't allocate a fresh
+ * routerstatus but use <b>vote_rs</b> instead.
+ *
+ * If <b>consensus_method</b> is nonzero, this routerstatus is part of a
+ * consensus, and we should parse it according to the method used to
+ * make that consensus.
+ *
+ * Parse according to the syntax used by the consensus flavor <b>flav</b>.
+ **/
+STATIC routerstatus_t *
+routerstatus_parse_entry_from_string(memarea_t *area,
+                                     const char **s, smartlist_t *tokens,
+                                     networkstatus_t *vote,
+                                     vote_routerstatus_t *vote_rs,
+                                     int consensus_method,
+                                     consensus_flavor_t flav)
+{
+  const char *eos, *s_dup = *s;
+  routerstatus_t *rs = NULL;
+  directory_token_t *tok;
+  char timebuf[ISO_TIME_LEN+1];
+  struct in_addr in;
+  int offset = 0;
+  tor_assert(tokens);
+  tor_assert(bool_eq(vote, vote_rs));
+
+  if (!consensus_method)
+    flav = FLAV_NS;
+  tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
+
+  eos = find_start_of_next_routerstatus(*s);
+
+  if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) {
+    log_warn(LD_DIR, "Error tokenizing router status");
+    goto err;
+  }
+  if (smartlist_len(tokens) < 1) {
+    log_warn(LD_DIR, "Impossibly short router status");
+    goto err;
+  }
+  tok = find_by_keyword(tokens, K_R);
+  tor_assert(tok->n_args >= 7); /* guaranteed by GE(7) in K_R setup */
+  if (flav == FLAV_NS) {
+    if (tok->n_args < 8) {
+      log_warn(LD_DIR, "Too few arguments to r");
+      goto err;
+    }
+  } else if (flav == FLAV_MICRODESC) {
+    offset = -1; /* There is no descriptor digest in an md consensus r line */
+  }
+
+  if (vote_rs) {
+    rs = &vote_rs->status;
+  } else {
+    rs = tor_malloc_zero(sizeof(routerstatus_t));
+  }
+
+  if (!is_legal_nickname(tok->args[0])) {
+    log_warn(LD_DIR,
+             "Invalid nickname %s in router status; skipping.",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  strlcpy(rs->nickname, tok->args[0], sizeof(rs->nickname));
+
+  if (digest_from_base64(rs->identity_digest, tok->args[1])) {
+    log_warn(LD_DIR, "Error decoding identity digest %s",
+             escaped(tok->args[1]));
+    goto err;
+  }
+
+  if (flav == FLAV_NS) {
+    if (digest_from_base64(rs->descriptor_digest, tok->args[2])) {
+      log_warn(LD_DIR, "Error decoding descriptor digest %s",
+               escaped(tok->args[2]));
+      goto err;
+    }
+  }
+
+  if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s",
+                   tok->args[3+offset], tok->args[4+offset]) < 0 ||
+      parse_iso_time(timebuf, &rs->published_on)<0) {
+    log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]",
+             tok->args[3+offset], tok->args[4+offset],
+             offset, (int)flav);
+    goto err;
+  }
+
+  if (tor_inet_aton(tok->args[5+offset], &in) == 0) {
+    log_warn(LD_DIR, "Error parsing router address in network-status %s",
+             escaped(tok->args[5+offset]));
+    goto err;
+  }
+  rs->addr = ntohl(in.s_addr);
+
+  rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset],
+                                         10,0,65535,NULL,NULL);
+  rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset],
+                                           10,0,65535,NULL,NULL);
+
+  {
+    smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+    if (a_lines) {
+      find_single_ipv6_orport(a_lines, &rs->ipv6_addr, &rs->ipv6_orport);
+      smartlist_free(a_lines);
+    }
+  }
+
+  tok = find_opt_by_keyword(tokens, K_S);
+  if (tok && vote) {
+    int i;
+    vote_rs->flags = 0;
+    for (i=0; i < tok->n_args; ++i) {
+      int p = smartlist_string_pos(vote->known_flags, tok->args[i]);
+      if (p >= 0) {
+        vote_rs->flags |= (UINT64_C(1)<<p);
+      } else {
+        log_warn(LD_DIR, "Flags line had a flag %s not listed in known_flags.",
+                 escaped(tok->args[i]));
+        goto err;
+      }
+    }
+  } else if (tok) {
+    /* This is a consensus, not a vote. */
+    int i;
+    for (i=0; i < tok->n_args; ++i) {
+      if (!strcmp(tok->args[i], "Exit"))
+        rs->is_exit = 1;
+      else if (!strcmp(tok->args[i], "Stable"))
+        rs->is_stable = 1;
+      else if (!strcmp(tok->args[i], "Fast"))
+        rs->is_fast = 1;
+      else if (!strcmp(tok->args[i], "Running"))
+        rs->is_flagged_running = 1;
+      else if (!strcmp(tok->args[i], "Named"))
+        rs->is_named = 1;
+      else if (!strcmp(tok->args[i], "Valid"))
+        rs->is_valid = 1;
+      else if (!strcmp(tok->args[i], "Guard"))
+        rs->is_possible_guard = 1;
+      else if (!strcmp(tok->args[i], "BadExit"))
+        rs->is_bad_exit = 1;
+      else if (!strcmp(tok->args[i], "Authority"))
+        rs->is_authority = 1;
+      else if (!strcmp(tok->args[i], "Unnamed") &&
+               consensus_method >= 2) {
+        /* Unnamed is computed right by consensus method 2 and later. */
+        rs->is_unnamed = 1;
+      } else if (!strcmp(tok->args[i], "HSDir")) {
+        rs->is_hs_dir = 1;
+      } else if (!strcmp(tok->args[i], "V2Dir")) {
+        rs->is_v2_dir = 1;
+      }
+    }
+    /* These are implied true by having been included in a consensus made
+     * with a given method */
+    rs->is_flagged_running = 1; /* Starting with consensus method 4. */
+    rs->is_valid = 1; /* Starting with consensus method 24. */
+  }
+  {
+    const char *protocols = NULL, *version = NULL;
+    if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
+      tor_assert(tok->n_args == 1);
+      protocols = tok->args[0];
+    }
+    if ((tok = find_opt_by_keyword(tokens, K_V))) {
+      tor_assert(tok->n_args == 1);
+      version = tok->args[0];
+      if (vote_rs) {
+        vote_rs->version = tor_strdup(tok->args[0]);
+      }
+    }
+
+    summarize_protover_flags(&rs->pv, protocols, version);
+  }
+
+  /* handle weighting/bandwidth info */
+  if ((tok = find_opt_by_keyword(tokens, K_W))) {
+    int i;
+    for (i=0; i < tok->n_args; ++i) {
+      if (!strcmpstart(tok->args[i], "Bandwidth=")) {
+        int ok;
+        rs->bandwidth_kb =
+          (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
+                                    10, 0, UINT32_MAX,
+                                    &ok, NULL);
+        if (!ok) {
+          log_warn(LD_DIR, "Invalid Bandwidth %s", escaped(tok->args[i]));
+          goto err;
+        }
+        rs->has_bandwidth = 1;
+      } else if (!strcmpstart(tok->args[i], "Measured=") && vote_rs) {
+        int ok;
+        vote_rs->measured_bw_kb =
+            (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
+                                      10, 0, UINT32_MAX, &ok, NULL);
+        if (!ok) {
+          log_warn(LD_DIR, "Invalid Measured Bandwidth %s",
+                   escaped(tok->args[i]));
+          goto err;
+        }
+        vote_rs->has_measured_bw = 1;
+        vote->has_measured_bws = 1;
+      } else if (!strcmpstart(tok->args[i], "Unmeasured=1")) {
+        rs->bw_is_unmeasured = 1;
+      } else if (!strcmpstart(tok->args[i], "GuardFraction=")) {
+        if (routerstatus_parse_guardfraction(tok->args[i],
+                                             vote, vote_rs, rs) < 0) {
+          goto err;
+        }
+      }
+    }
+  }
+
+  /* parse exit policy summaries */
+  if ((tok = find_opt_by_keyword(tokens, K_P))) {
+    tor_assert(tok->n_args == 1);
+    if (strcmpstart(tok->args[0], "accept ") &&
+        strcmpstart(tok->args[0], "reject ")) {
+      log_warn(LD_DIR, "Unknown exit policy summary type %s.",
+               escaped(tok->args[0]));
+      goto err;
+    }
+    /* XXX weasel: parse this into ports and represent them somehow smart,
+     * maybe not here but somewhere on if we need it for the client.
+     * we should still parse it here to check it's valid tho.
+     */
+    rs->exitsummary = tor_strdup(tok->args[0]);
+    rs->has_exitsummary = 1;
+  }
+
+  if (vote_rs) {
+    SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) {
+      if (t->tp == K_M && t->n_args) {
+        vote_microdesc_hash_t *line =
+          tor_malloc(sizeof(vote_microdesc_hash_t));
+        line->next = vote_rs->microdesc;
+        line->microdesc_hash_line = tor_strdup(t->args[0]);
+        vote_rs->microdesc = line;
+      }
+      if (t->tp == K_ID) {
+        tor_assert(t->n_args >= 2);
+        if (!strcmp(t->args[0], "ed25519")) {
+          vote_rs->has_ed25519_listing = 1;
+          if (strcmp(t->args[1], "none") &&
+              digest256_from_base64((char*)vote_rs->ed25519_id,
+                                    t->args[1])<0) {
+            log_warn(LD_DIR, "Bogus ed25519 key in networkstatus vote");
+            goto err;
+          }
+        }
+      }
+      if (t->tp == K_PROTO) {
+        tor_assert(t->n_args == 1);
+        vote_rs->protocols = tor_strdup(t->args[0]);
+      }
+    } SMARTLIST_FOREACH_END(t);
+  } else if (flav == FLAV_MICRODESC) {
+    tok = find_opt_by_keyword(tokens, K_M);
+    if (tok) {
+      tor_assert(tok->n_args);
+      if (digest256_from_base64(rs->descriptor_digest, tok->args[0])) {
+        log_warn(LD_DIR, "Error decoding microdescriptor digest %s",
+                 escaped(tok->args[0]));
+        goto err;
+      }
+    } else {
+      log_info(LD_BUG, "Found an entry in networkstatus with no "
+               "microdescriptor digest. (Router %s ($%s) at %s:%d.)",
+               rs->nickname, hex_str(rs->identity_digest, DIGEST_LEN),
+               fmt_addr32(rs->addr), rs->or_port);
+    }
+  }
+
+  if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME))
+    rs->is_named = 0;
+
+  goto done;
+ err:
+  dump_desc(s_dup, "routerstatus entry");
+  if (rs && !vote_rs)
+    routerstatus_free(rs);
+  rs = NULL;
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_clear(tokens);
+  if (area) {
+    DUMP_AREA(area, "routerstatus entry");
+    memarea_clear(area);
+  }
+  *s = eos;
+
+  return rs;
+}
+
+int
+compare_vote_routerstatus_entries(const void **_a, const void **_b)
+{
+  const vote_routerstatus_t *a = *_a, *b = *_b;
+  return fast_memcmp(a->status.identity_digest, b->status.identity_digest,
+                     DIGEST_LEN);
+}
+
+/** Verify the bandwidth weights of a network status document */
+int
+networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
+{
+  int64_t G=0, M=0, E=0, D=0, T=0;
+  double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed;
+  double Gtotal=0, Mtotal=0, Etotal=0;
+  const char *casename = NULL;
+  int valid = 1;
+  (void) consensus_method;
+
+  const int64_t weight_scale = networkstatus_get_weight_scale_param(ns);
+  tor_assert(weight_scale >= 1);
+  Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1);
+  Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1);
+  Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1);
+  Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1);
+  Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1);
+  Wme = networkstatus_get_bw_weight(ns, "Wme", -1);
+  Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1);
+  Weg = networkstatus_get_bw_weight(ns, "Weg", -1);
+  Wem = networkstatus_get_bw_weight(ns, "Wem", -1);
+  Wee = networkstatus_get_bw_weight(ns, "Wee", -1);
+  Wed = networkstatus_get_bw_weight(ns, "Wed", -1);
+
+  if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0
+          || Wem<0 || Wee<0 || Wed<0) {
+    log_warn(LD_BUG, "No bandwidth weights produced in consensus!");
+    return 0;
+  }
+
+  // First, sanity check basic summing properties that hold for all cases
+  // We use > 1 as the check for these because they are computed as integers.
+  // Sometimes there are rounding errors.
+  if (fabs(Wmm - weight_scale) > 1) {
+    log_warn(LD_BUG, "Wmm=%f != %"PRId64,
+             Wmm, (weight_scale));
+    valid = 0;
+  }
+
+  if (fabs(Wem - Wee) > 1) {
+    log_warn(LD_BUG, "Wem=%f != Wee=%f", Wem, Wee);
+    valid = 0;
+  }
+
+  if (fabs(Wgm - Wgg) > 1) {
+    log_warn(LD_BUG, "Wgm=%f != Wgg=%f", Wgm, Wgg);
+    valid = 0;
+  }
+
+  if (fabs(Weg - Wed) > 1) {
+    log_warn(LD_BUG, "Wed=%f != Weg=%f", Wed, Weg);
+    valid = 0;
+  }
+
+  if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) {
+    log_warn(LD_BUG, "Wgg=%f != %"PRId64" - Wmg=%f", Wgg,
+             (weight_scale), Wmg);
+    valid = 0;
+  }
+
+  if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) {
+    log_warn(LD_BUG, "Wee=%f != %"PRId64" - Wme=%f", Wee,
+             (weight_scale), Wme);
+    valid = 0;
+  }
+
+  if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) {
+    log_warn(LD_BUG, "Wgd=%f + Wmd=%f + Wed=%f != %"PRId64,
+             Wgd, Wmd, Wed, (weight_scale));
+    valid = 0;
+  }
+
+  Wgg /= weight_scale;
+  Wgm /= weight_scale; (void) Wgm; // unused from here on.
+  Wgd /= weight_scale;
+
+  Wmg /= weight_scale;
+  Wmm /= weight_scale;
+  Wme /= weight_scale;
+  Wmd /= weight_scale;
+
+  Weg /= weight_scale; (void) Weg; // unused from here on.
+  Wem /= weight_scale; (void) Wem; // unused from here on.
+  Wee /= weight_scale;
+  Wed /= weight_scale;
+
+  // Then, gather G, M, E, D, T to determine case
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    int is_exit = 0;
+    /* Bug #2203: Don't count bad exits as exits for balancing */
+    is_exit = rs->is_exit && !rs->is_bad_exit;
+    if (rs->has_bandwidth) {
+      T += rs->bandwidth_kb;
+      if (is_exit && rs->is_possible_guard) {
+        D += rs->bandwidth_kb;
+        Gtotal += Wgd*rs->bandwidth_kb;
+        Mtotal += Wmd*rs->bandwidth_kb;
+        Etotal += Wed*rs->bandwidth_kb;
+      } else if (is_exit) {
+        E += rs->bandwidth_kb;
+        Mtotal += Wme*rs->bandwidth_kb;
+        Etotal += Wee*rs->bandwidth_kb;
+      } else if (rs->is_possible_guard) {
+        G += rs->bandwidth_kb;
+        Gtotal += Wgg*rs->bandwidth_kb;
+        Mtotal += Wmg*rs->bandwidth_kb;
+      } else {
+        M += rs->bandwidth_kb;
+        Mtotal += Wmm*rs->bandwidth_kb;
+      }
+    } else {
+      log_warn(LD_BUG, "Missing consensus bandwidth for router %s",
+               routerstatus_describe(rs));
+    }
+  } SMARTLIST_FOREACH_END(rs);
+
+  // Finally, check equality conditions depending upon case 1, 2 or 3
+  // Full equality cases: 1, 3b
+  // Partial equality cases: 2b (E=G), 3a (M=E)
+  // Fully unknown: 2a
+  if (3*E >= T && 3*G >= T) {
+    // Case 1: Neither are scarce
+    casename = "Case 1";
+    if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
+      log_warn(LD_DIR,
+               "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
+               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+               " T=%"PRId64". "
+               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+               casename, Etotal, Mtotal,
+               (G), (M), (E),
+               (D), (T),
+               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+      valid = 0;
+    }
+    if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+      log_warn(LD_DIR,
+               "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+               " T=%"PRId64". "
+               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+               casename, Etotal, Gtotal,
+               (G), (M), (E),
+               (D), (T),
+               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+      valid = 0;
+    }
+    if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
+      log_warn(LD_DIR,
+               "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
+               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+               " T=%"PRId64". "
+               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+               casename, Mtotal, Gtotal,
+               (G), (M), (E),
+               (D), (T),
+               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+      valid = 0;
+    }
+  } else if (3*E < T && 3*G < T) {
+    int64_t R = MIN(E, G);
+    int64_t S = MAX(E, G);
+    /*
+     * Case 2: Both Guards and Exits are scarce
+     * Balance D between E and G, depending upon
+     * D capacity and scarcity. Devote no extra
+     * bandwidth to middle nodes.
+     */
+    if (R+D < S) { // Subcase a
+      double Rtotal, Stotal;
+      if (E < G) {
+        Rtotal = Etotal;
+        Stotal = Gtotal;
+      } else {
+        Rtotal = Gtotal;
+        Stotal = Etotal;
+      }
+      casename = "Case 2a";
+      // Rtotal < Stotal
+      if (Rtotal > Stotal) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Rtotal %f > Stotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Rtotal, Stotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      // Rtotal < T/3
+      if (3*Rtotal > T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Rtotal %f > T "
+                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
+                   " D=%"PRId64" T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Rtotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      // Stotal < T/3
+      if (3*Stotal > T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Stotal %f > T "
+                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
+                   " D=%"PRId64" T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Stotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      // Mtotal > T/3
+      if (3*Mtotal < T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Mtotal %f < T "
+                   "%"PRId64". "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Mtotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+    } else { // Subcase b: R+D > S
+      casename = "Case 2b";
+
+      /* Check the rare-M redirect case. */
+      if (D != 0 && 3*M < T) {
+        casename = "Case 2b (balanced)";
+        if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Etotal, Mtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Etotal, Gtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+        if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Mtotal, Gtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      } else {
+        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Etotal, Gtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      }
+    }
+  } else { // if (E < T/3 || G < T/3) {
+    int64_t S = MIN(E, G);
+    int64_t NS = MAX(E, G);
+    if (3*(S+D) < T) { // Subcase a:
+      double Stotal;
+      double NStotal;
+      if (G < E) {
+        casename = "Case 3a (G scarce)";
+        Stotal = Gtotal;
+        NStotal = Etotal;
+      } else { // if (G >= E) {
+        casename = "Case 3a (E scarce)";
+        NStotal = Gtotal;
+        Stotal = Etotal;
+      }
+      // Stotal < T/3
+      if (3*Stotal > T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Stotal %f > T "
+                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
+                   " D=%"PRId64" T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Stotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      if (NS >= M) {
+        if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: NStotal %f != Mtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, NStotal, Mtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      } else {
+        // if NS < M, NStotal > T/3 because only one of G or E is scarce
+        if (3*NStotal < T) {
+          log_warn(LD_DIR,
+                     "Bw Weight Failure for %s: 3*NStotal %f < T "
+                     "%"PRId64". G=%"PRId64" M=%"PRId64
+                     " E=%"PRId64" D=%"PRId64" T=%"PRId64". "
+                     "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                     casename, NStotal*3, (T),
+                     (G), (M), (E),
+                     (D), (T),
+                     Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      }
+    } else { // Subcase b: S+D >= T/3
+      casename = "Case 3b";
+      if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
+        log_warn(LD_DIR,
+                 "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
+                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                 " T=%"PRId64". "
+                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                 casename, Etotal, Mtotal,
+                 (G), (M), (E),
+                 (D), (T),
+                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+        log_warn(LD_DIR,
+                 "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                 " T=%"PRId64". "
+                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                 casename, Etotal, Gtotal,
+                 (G), (M), (E),
+                 (D), (T),
+                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
+        log_warn(LD_DIR,
+                 "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
+                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                 " T=%"PRId64". "
+                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                 casename, Mtotal, Gtotal,
+                 (G), (M), (E),
+                 (D), (T),
+                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+    }
+  }
+
+  if (valid)
+    log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.",
+               casename);
+
+  return valid;
+}
+
+/** Check if a shared random value of type <b>srv_type</b> is in
+ *  <b>tokens</b>. If there is, parse it and set it to <b>srv_out</b>. Return
+ *  -1 on failure, 0 on success. The resulting srv is allocated on the heap and
+ *  it's the responsibility of the caller to free it. */
+static int
+extract_one_srv(smartlist_t *tokens, directory_keyword srv_type,
+                sr_srv_t **srv_out)
+{
+  int ret = -1;
+  directory_token_t *tok;
+  sr_srv_t *srv = NULL;
+  smartlist_t *chunks;
+
+  tor_assert(tokens);
+
+  chunks = smartlist_new();
+  tok = find_opt_by_keyword(tokens, srv_type);
+  if (!tok) {
+    /* That's fine, no SRV is allowed. */
+    ret = 0;
+    goto end;
+  }
+  for (int i = 0; i < tok->n_args; i++) {
+    smartlist_add(chunks, tok->args[i]);
+  }
+  srv = sr_parse_srv(chunks);
+  if (srv == NULL) {
+    log_warn(LD_DIR, "SR: Unparseable SRV %s", escaped(tok->object_body));
+    goto end;
+  }
+  /* All is good. */
+  *srv_out = srv;
+  ret = 0;
+ end:
+  smartlist_free(chunks);
+  return ret;
+}
+
+/** Extract any shared random values found in <b>tokens</b> and place them in
+ *  the networkstatus <b>ns</b>. */
+static void
+extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
+{
+  const char *voter_identity;
+  networkstatus_voter_info_t *voter;
+
+  tor_assert(ns);
+  tor_assert(tokens);
+  /* Can be only one of them else code flow. */
+  tor_assert(ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS);
+
+  if (ns->type == NS_TYPE_VOTE) {
+    voter = smartlist_get(ns->voters, 0);
+    tor_assert(voter);
+    voter_identity = hex_str(voter->identity_digest,
+                             sizeof(voter->identity_digest));
+  } else {
+    /* Consensus has multiple voters so no specific voter. */
+    voter_identity = "consensus";
+  }
+
+  /* We extract both, and on error everything is stopped because it means
+   * the vote is malformed for the shared random value(s). */
+  if (extract_one_srv(tokens, K_PREVIOUS_SRV, &ns->sr_info.previous_srv) < 0) {
+    log_warn(LD_DIR, "SR: Unable to parse previous SRV from %s",
+             voter_identity);
+    /* Maybe we have a chance with the current SRV so let's try it anyway. */
+  }
+  if (extract_one_srv(tokens, K_CURRENT_SRV, &ns->sr_info.current_srv) < 0) {
+    log_warn(LD_DIR, "SR: Unable to parse current SRV from %s",
+             voter_identity);
+  }
+}
+
+/** Parse a v3 networkstatus vote, opinion, or consensus (depending on
+ * ns_type), from <b>s</b>, and return the result.  Return NULL on failure. */
+networkstatus_t *
+networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
+                                     networkstatus_type_t ns_type)
+{
+  smartlist_t *tokens = smartlist_new();
+  smartlist_t *rs_tokens = NULL, *footer_tokens = NULL;
+  networkstatus_voter_info_t *voter = NULL;
+  networkstatus_t *ns = NULL;
+  common_digests_t ns_digests;
+  uint8_t sha3_as_signed[DIGEST256_LEN];
+  const char *cert, *end_of_header, *end_of_footer, *s_dup = s;
+  directory_token_t *tok;
+  struct in_addr in;
+  int i, inorder, n_signatures = 0;
+  memarea_t *area = NULL, *rs_area = NULL;
+  consensus_flavor_t flav = FLAV_NS;
+  char *last_kwd=NULL;
+
+  tor_assert(s);
+
+  if (eos_out)
+    *eos_out = NULL;
+
+  if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
+      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
+    log_warn(LD_DIR, "Unable to compute digest of network-status");
+    goto err;
+  }
+
+  area = memarea_new();
+  end_of_header = find_start_of_next_routerstatus(s);
+  if (tokenize_string(area, s, end_of_header, tokens,
+                      (ns_type == NS_TYPE_CONSENSUS) ?
+                      networkstatus_consensus_token_table :
+                      networkstatus_token_table, 0)) {
+    log_warn(LD_DIR, "Error tokenizing network-status header");
+    goto err;
+  }
+
+  ns = tor_malloc_zero(sizeof(networkstatus_t));
+  memcpy(&ns->digests, &ns_digests, sizeof(ns_digests));
+  memcpy(&ns->digest_sha3_as_signed, sha3_as_signed, sizeof(sha3_as_signed));
+
+  tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION);
+  tor_assert(tok);
+  if (tok->n_args > 1) {
+    int flavor = networkstatus_parse_flavor_name(tok->args[1]);
+    if (flavor < 0) {
+      log_warn(LD_DIR, "Can't parse document with unknown flavor %s",
+               escaped(tok->args[1]));
+      goto err;
+    }
+    ns->flavor = flav = flavor;
+  }
+  if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) {
+    log_warn(LD_DIR, "Flavor found on non-consensus networkstatus.");
+    goto err;
+  }
+
+  if (ns_type != NS_TYPE_CONSENSUS) {
+    const char *end_of_cert = NULL;
+    if (!(cert = strstr(s, "\ndir-key-certificate-version")))
+      goto err;
+    ++cert;
+    ns->cert = authority_cert_parse_from_string(cert, &end_of_cert);
+    if (!ns->cert || !end_of_cert || end_of_cert > end_of_header)
+      goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_VOTE_STATUS);
+  tor_assert(tok->n_args);
+  if (!strcmp(tok->args[0], "vote")) {
+    ns->type = NS_TYPE_VOTE;
+  } else if (!strcmp(tok->args[0], "consensus")) {
+    ns->type = NS_TYPE_CONSENSUS;
+  } else if (!strcmp(tok->args[0], "opinion")) {
+    ns->type = NS_TYPE_OPINION;
+  } else {
+    log_warn(LD_DIR, "Unrecognized vote status %s in network-status",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  if (ns_type != ns->type) {
+    log_warn(LD_DIR, "Got the wrong kind of v3 networkstatus.");
+    goto err;
+  }
+
+  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) {
+    tok = find_by_keyword(tokens, K_PUBLISHED);
+    if (parse_iso_time(tok->args[0], &ns->published))
+      goto err;
+
+    ns->supported_methods = smartlist_new();
+    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHODS);
+    if (tok) {
+      for (i=0; i < tok->n_args; ++i)
+        smartlist_add_strdup(ns->supported_methods, tok->args[i]);
+    } else {
+      smartlist_add_strdup(ns->supported_methods, "1");
+    }
+  } else {
+    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHOD);
+    if (tok) {
+      int num_ok;
+      ns->consensus_method = (int)tor_parse_long(tok->args[0], 10, 1, INT_MAX,
+                                                 &num_ok, NULL);
+      if (!num_ok)
+        goto err;
+    } else {
+      ns->consensus_method = 1;
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_CLIENT_PROTOCOLS)))
+    ns->recommended_client_protocols = tor_strdup(tok->args[0]);
+  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_RELAY_PROTOCOLS)))
+    ns->recommended_relay_protocols = tor_strdup(tok->args[0]);
+  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_CLIENT_PROTOCOLS)))
+    ns->required_client_protocols = tor_strdup(tok->args[0]);
+  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_RELAY_PROTOCOLS)))
+    ns->required_relay_protocols = tor_strdup(tok->args[0]);
+
+  tok = find_by_keyword(tokens, K_VALID_AFTER);
+  if (parse_iso_time(tok->args[0], &ns->valid_after))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_FRESH_UNTIL);
+  if (parse_iso_time(tok->args[0], &ns->fresh_until))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_VALID_UNTIL);
+  if (parse_iso_time(tok->args[0], &ns->valid_until))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_VOTING_DELAY);
+  tor_assert(tok->n_args >= 2);
+  {
+    int ok;
+    ns->vote_seconds =
+      (int) tor_parse_long(tok->args[0], 10, 0, INT_MAX, &ok, NULL);
+    if (!ok)
+      goto err;
+    ns->dist_seconds =
+      (int) tor_parse_long(tok->args[1], 10, 0, INT_MAX, &ok, NULL);
+    if (!ok)
+      goto err;
+  }
+  if (ns->valid_after +
+      (get_options()->TestingTorNetwork ?
+       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) > ns->fresh_until) {
+    log_warn(LD_DIR, "Vote/consensus freshness interval is too short");
+    goto err;
+  }
+  if (ns->valid_after +
+      (get_options()->TestingTorNetwork ?
+       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL)*2 > ns->valid_until) {
+    log_warn(LD_DIR, "Vote/consensus liveness interval is too short");
+    goto err;
+  }
+  if (ns->vote_seconds < MIN_VOTE_SECONDS) {
+    log_warn(LD_DIR, "Vote seconds is too short");
+    goto err;
+  }
+  if (ns->dist_seconds < MIN_DIST_SECONDS) {
+    log_warn(LD_DIR, "Dist seconds is too short");
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_CLIENT_VERSIONS))) {
+    ns->client_versions = tor_strdup(tok->args[0]);
+  }
+  if ((tok = find_opt_by_keyword(tokens, K_SERVER_VERSIONS))) {
+    ns->server_versions = tor_strdup(tok->args[0]);
+  }
+
+  {
+    smartlist_t *package_lst = find_all_by_keyword(tokens, K_PACKAGE);
+    ns->package_lines = smartlist_new();
+    if (package_lst) {
+      SMARTLIST_FOREACH(package_lst, directory_token_t *, t,
+                    smartlist_add_strdup(ns->package_lines, t->args[0]));
+    }
+    smartlist_free(package_lst);
+  }
+
+  tok = find_by_keyword(tokens, K_KNOWN_FLAGS);
+  ns->known_flags = smartlist_new();
+  inorder = 1;
+  for (i = 0; i < tok->n_args; ++i) {
+    smartlist_add_strdup(ns->known_flags, tok->args[i]);
+    if (i>0 && strcmp(tok->args[i-1], tok->args[i])>= 0) {
+      log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
+      inorder = 0;
+    }
+  }
+  if (!inorder) {
+    log_warn(LD_DIR, "known-flags not in order");
+    goto err;
+  }
+  if (ns->type != NS_TYPE_CONSENSUS &&
+      smartlist_len(ns->known_flags) > MAX_KNOWN_FLAGS_IN_VOTE) {
+    /* If we allowed more than 64 flags in votes, then parsing them would make
+     * us invoke undefined behavior whenever we used 1<<flagnum to do a
+     * bit-shift. This is only for votes and opinions: consensus users don't
+     * care about flags they don't recognize, and so don't build a bitfield
+     * for them. */
+    log_warn(LD_DIR, "Too many known-flags in consensus vote or opinion");
+    goto err;
+  }
+
+  tok = find_opt_by_keyword(tokens, K_PARAMS);
+  if (tok) {
+    int any_dups = 0;
+    inorder = 1;
+    ns->net_params = smartlist_new();
+    for (i = 0; i < tok->n_args; ++i) {
+      int ok=0;
+      char *eq = strchr(tok->args[i], '=');
+      size_t eq_pos;
+      if (!eq) {
+        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
+        goto err;
+      }
+      eq_pos = eq-tok->args[i];
+      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
+      if (!ok) {
+        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
+        goto err;
+      }
+      if (i > 0 && strcmp(tok->args[i-1], tok->args[i]) >= 0) {
+        log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
+        inorder = 0;
+      }
+      if (last_kwd && eq_pos == strlen(last_kwd) &&
+          fast_memeq(last_kwd, tok->args[i], eq_pos)) {
+        log_warn(LD_DIR, "Duplicate value for %s parameter",
+                 escaped(tok->args[i]));
+        any_dups = 1;
+      }
+      tor_free(last_kwd);
+      last_kwd = tor_strndup(tok->args[i], eq_pos);
+      smartlist_add_strdup(ns->net_params, tok->args[i]);
+    }
+    if (!inorder) {
+      log_warn(LD_DIR, "params not in order");
+      goto err;
+    }
+    if (any_dups) {
+      log_warn(LD_DIR, "Duplicate in parameters");
+      goto err;
+    }
+  }
+
+  ns->voters = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
+    tok = _tok;
+    if (tok->tp == K_DIR_SOURCE) {
+      tor_assert(tok->n_args >= 6);
+
+      if (voter)
+        smartlist_add(ns->voters, voter);
+      voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
+      voter->sigs = smartlist_new();
+      if (ns->type != NS_TYPE_CONSENSUS)
+        memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN);
+
+      voter->nickname = tor_strdup(tok->args[0]);
+      if (strlen(tok->args[1]) != HEX_DIGEST_LEN ||
+          base16_decode(voter->identity_digest, sizeof(voter->identity_digest),
+                        tok->args[1], HEX_DIGEST_LEN)
+                        != sizeof(voter->identity_digest)) {
+        log_warn(LD_DIR, "Error decoding identity digest %s in "
+                 "network-status document.", escaped(tok->args[1]));
+        goto err;
+      }
+      if (ns->type != NS_TYPE_CONSENSUS &&
+          tor_memneq(ns->cert->cache_info.identity_digest,
+                 voter->identity_digest, DIGEST_LEN)) {
+        log_warn(LD_DIR,"Mismatch between identities in certificate and vote");
+        goto err;
+      }
+      if (ns->type != NS_TYPE_CONSENSUS) {
+        if (authority_cert_is_blacklisted(ns->cert)) {
+          log_warn(LD_DIR, "Rejecting vote signature made with blacklisted "
+                   "signing key %s",
+                   hex_str(ns->cert->signing_key_digest, DIGEST_LEN));
+          goto err;
+        }
+      }
+      voter->address = tor_strdup(tok->args[2]);
+      if (!tor_inet_aton(tok->args[3], &in)) {
+        log_warn(LD_DIR, "Error decoding IP address %s in network-status.",
+                 escaped(tok->args[3]));
+        goto err;
+      }
+      voter->addr = ntohl(in.s_addr);
+      int ok;
+      voter->dir_port = (uint16_t)
+        tor_parse_long(tok->args[4], 10, 0, 65535, &ok, NULL);
+      if (!ok)
+        goto err;
+      voter->or_port = (uint16_t)
+        tor_parse_long(tok->args[5], 10, 0, 65535, &ok, NULL);
+      if (!ok)
+        goto err;
+    } else if (tok->tp == K_CONTACT) {
+      if (!voter || voter->contact) {
+        log_warn(LD_DIR, "contact element is out of place.");
+        goto err;
+      }
+      voter->contact = tor_strdup(tok->args[0]);
+    } else if (tok->tp == K_VOTE_DIGEST) {
+      tor_assert(ns->type == NS_TYPE_CONSENSUS);
+      tor_assert(tok->n_args >= 1);
+      if (!voter || ! tor_digest_is_zero(voter->vote_digest)) {
+        log_warn(LD_DIR, "vote-digest element is out of place.");
+        goto err;
+      }
+      if (strlen(tok->args[0]) != HEX_DIGEST_LEN ||
+        base16_decode(voter->vote_digest, sizeof(voter->vote_digest),
+                      tok->args[0], HEX_DIGEST_LEN)
+                      != sizeof(voter->vote_digest)) {
+        log_warn(LD_DIR, "Error decoding vote digest %s in "
+                 "network-status consensus.", escaped(tok->args[0]));
+        goto err;
+      }
+    }
+  } SMARTLIST_FOREACH_END(_tok);
+  if (voter) {
+    smartlist_add(ns->voters, voter);
+    voter = NULL;
+  }
+  if (smartlist_len(ns->voters) == 0) {
+    log_warn(LD_DIR, "Missing dir-source elements in a networkstatus.");
+    goto err;
+  } else if (ns->type != NS_TYPE_CONSENSUS && smartlist_len(ns->voters) != 1) {
+    log_warn(LD_DIR, "Too many dir-source elements in a vote networkstatus.");
+    goto err;
+  }
+
+  if (ns->type != NS_TYPE_CONSENSUS &&
+      (tok = find_opt_by_keyword(tokens, K_LEGACY_DIR_KEY))) {
+    int bad = 1;
+    if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
+      networkstatus_voter_info_t *voter_0 = smartlist_get(ns->voters, 0);
+      if (base16_decode(voter_0->legacy_id_digest, DIGEST_LEN,
+                        tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN)
+        bad = 1;
+      else
+        bad = 0;
+    }
+    if (bad) {
+      log_warn(LD_DIR, "Invalid legacy key digest %s on vote.",
+               escaped(tok->args[0]));
+    }
+  }
+
+  /* If this is a vote document, check if information about the shared
+     randomness protocol is included, and extract it. */
+  if (ns->type == NS_TYPE_VOTE) {
+    dirvote_parse_sr_commits(ns, tokens);
+  }
+  /* For both a vote and consensus, extract the shared random values. */
+  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS) {
+    extract_shared_random_srvs(ns, tokens);
+  }
+
+  /* Parse routerstatus lines. */
+  rs_tokens = smartlist_new();
+  rs_area = memarea_new();
+  s = end_of_header;
+  ns->routerstatus_list = smartlist_new();
+
+  while (!strcmpstart(s, "r ")) {
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
+                                               rs, 0, 0)) {
+        smartlist_add(ns->routerstatus_list, rs);
+      } else {
+        vote_routerstatus_free(rs);
+      }
+    } else {
+      routerstatus_t *rs;
+      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens,
+                                                     NULL, NULL,
+                                                     ns->consensus_method,
+                                                     flav))) {
+        /* Use exponential-backoff scheduling when downloading microdescs */
+        smartlist_add(ns->routerstatus_list, rs);
+      }
+    }
+  }
+  for (i = 1; i < smartlist_len(ns->routerstatus_list); ++i) {
+    routerstatus_t *rs1, *rs2;
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      vote_routerstatus_t *a = smartlist_get(ns->routerstatus_list, i-1);
+      vote_routerstatus_t *b = smartlist_get(ns->routerstatus_list, i);
+      rs1 = &a->status; rs2 = &b->status;
+    } else {
+      rs1 = smartlist_get(ns->routerstatus_list, i-1);
+      rs2 = smartlist_get(ns->routerstatus_list, i);
+    }
+    if (fast_memcmp(rs1->identity_digest, rs2->identity_digest, DIGEST_LEN)
+        >= 0) {
+      log_warn(LD_DIR, "Networkstatus entries not sorted by identity digest");
+      goto err;
+    }
+  }
+  if (ns_type != NS_TYPE_CONSENSUS) {
+    digest256map_t *ed_id_map = digest256map_new();
+    SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *,
+                            vrs) {
+      if (! vrs->has_ed25519_listing ||
+          tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
+        continue;
+      if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) {
+        log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not "
+                 "unique");
+        digest256map_free(ed_id_map, NULL);
+        goto err;
+      }
+      digest256map_set(ed_id_map, vrs->ed25519_id, (void*)1);
+    } SMARTLIST_FOREACH_END(vrs);
+    digest256map_free(ed_id_map, NULL);
+  }
+
+  /* Parse footer; check signature. */
+  footer_tokens = smartlist_new();
+  if ((end_of_footer = strstr(s, "\nnetwork-status-version ")))
+    ++end_of_footer;
+  else
+    end_of_footer = s + strlen(s);
+  if (tokenize_string(area,s, end_of_footer, footer_tokens,
+                      networkstatus_vote_footer_token_table, 0)) {
+    log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
+    goto err;
+  }
+
+  {
+    int found_sig = 0;
+    SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
+      tok = _tok;
+      if (tok->tp == K_DIRECTORY_SIGNATURE)
+        found_sig = 1;
+      else if (found_sig) {
+        log_warn(LD_DIR, "Extraneous token after first directory-signature");
+        goto err;
+      }
+    } SMARTLIST_FOREACH_END(_tok);
+  }
+
+  if ((tok = find_opt_by_keyword(footer_tokens, K_DIRECTORY_FOOTER))) {
+    if (tok != smartlist_get(footer_tokens, 0)) {
+      log_warn(LD_DIR, "Misplaced directory-footer token");
+      goto err;
+    }
+  }
+
+  tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS);
+  if (tok) {
+    ns->weight_params = smartlist_new();
+    for (i = 0; i < tok->n_args; ++i) {
+      int ok=0;
+      char *eq = strchr(tok->args[i], '=');
+      if (!eq) {
+        log_warn(LD_DIR, "Bad element '%s' in weight params",
+                 escaped(tok->args[i]));
+        goto err;
+      }
+      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
+      if (!ok) {
+        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
+        goto err;
+      }
+      smartlist_add_strdup(ns->weight_params, tok->args[i]);
+    }
+  }
+
+  SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
+    char declared_identity[DIGEST_LEN];
+    networkstatus_voter_info_t *v;
+    document_signature_t *sig;
+    const char *id_hexdigest = NULL;
+    const char *sk_hexdigest = NULL;
+    digest_algorithm_t alg = DIGEST_SHA1;
+    tok = _tok;
+    if (tok->tp != K_DIRECTORY_SIGNATURE)
+      continue;
+    tor_assert(tok->n_args >= 2);
+    if (tok->n_args == 2) {
+      id_hexdigest = tok->args[0];
+      sk_hexdigest = tok->args[1];
+    } else {
+      const char *algname = tok->args[0];
+      int a;
+      id_hexdigest = tok->args[1];
+      sk_hexdigest = tok->args[2];
+      a = crypto_digest_algorithm_parse_name(algname);
+      if (a<0) {
+        log_warn(LD_DIR, "Unknown digest algorithm %s; skipping",
+                 escaped(algname));
+        continue;
+      }
+      alg = a;
+    }
+
+    if (!tok->object_type ||
+        strcmp(tok->object_type, "SIGNATURE") ||
+        tok->object_size < 128 || tok->object_size > 512) {
+      log_warn(LD_DIR, "Bad object type or length on directory-signature");
+      goto err;
+    }
+
+    if (strlen(id_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(declared_identity, sizeof(declared_identity),
+                      id_hexdigest, HEX_DIGEST_LEN)
+                      != sizeof(declared_identity)) {
+      log_warn(LD_DIR, "Error decoding declared identity %s in "
+               "network-status document.", escaped(id_hexdigest));
+      goto err;
+    }
+    if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) {
+      log_warn(LD_DIR, "ID on signature on network-status document does "
+               "not match any declared directory source.");
+      goto err;
+    }
+    sig = tor_malloc_zero(sizeof(document_signature_t));
+    memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN);
+    sig->alg = alg;
+    if (strlen(sk_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest),
+                      sk_hexdigest, HEX_DIGEST_LEN)
+                      != sizeof(sig->signing_key_digest)) {
+      log_warn(LD_DIR, "Error decoding declared signing key digest %s in "
+               "network-status document.", escaped(sk_hexdigest));
+      tor_free(sig);
+      goto err;
+    }
+
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      if (tor_memneq(declared_identity, ns->cert->cache_info.identity_digest,
+                 DIGEST_LEN)) {
+        log_warn(LD_DIR, "Digest mismatch between declared and actual on "
+                 "network-status vote.");
+        tor_free(sig);
+        goto err;
+      }
+    }
+
+    if (networkstatus_get_voter_sig_by_alg(v, sig->alg)) {
+      /* We already parsed a vote with this algorithm from this voter. Use the
+         first one. */
+      log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus "
+             "that contains two signatures from the same voter with the same "
+             "algorithm. Ignoring the second signature.");
+      tor_free(sig);
+      continue;
+    }
+
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN,
+                                tok, ns->cert->signing_key, 0,
+                                "network-status document")) {
+        tor_free(sig);
+        goto err;
+      }
+      sig->good_signature = 1;
+    } else {
+      if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) {
+        tor_free(sig);
+        goto err;
+      }
+      sig->signature = tor_memdup(tok->object_body, tok->object_size);
+      sig->signature_len = (int) tok->object_size;
+    }
+    smartlist_add(v->sigs, sig);
+
+    ++n_signatures;
+  } SMARTLIST_FOREACH_END(_tok);
+
+  if (! n_signatures) {
+    log_warn(LD_DIR, "No signatures on networkstatus document.");
+    goto err;
+  } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) {
+    log_warn(LD_DIR, "Received more than one signature on a "
+             "network-status vote.");
+    goto err;
+  }
+
+  if (eos_out)
+    *eos_out = end_of_footer;
+
+  goto done;
+ err:
+  dump_desc(s_dup, "v3 networkstatus");
+  networkstatus_vote_free(ns);
+  ns = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (voter) {
+    if (voter->sigs) {
+      SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                        document_signature_free(sig));
+      smartlist_free(voter->sigs);
+    }
+    tor_free(voter->nickname);
+    tor_free(voter->address);
+    tor_free(voter->contact);
+    tor_free(voter);
+  }
+  if (rs_tokens) {
+    SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(rs_tokens);
+  }
+  if (footer_tokens) {
+    SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(footer_tokens);
+  }
+  if (area) {
+    DUMP_AREA(area, "v3 networkstatus");
+    memarea_drop_all(area);
+  }
+  if (rs_area)
+    memarea_drop_all(rs_area);
+  tor_free(last_kwd);
+
+  return ns;
+}
+
+/** Return the common_digests_t that holds the digests of the
+ * <b>flavor_name</b>-flavored networkstatus according to the detached
+ * signatures document <b>sigs</b>, allocating a new common_digests_t as
+ * needed. */
+static common_digests_t *
+detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name)
+{
+  common_digests_t *d = strmap_get(sigs->digests, flavor_name);
+  if (!d) {
+    d = tor_malloc_zero(sizeof(common_digests_t));
+    strmap_set(sigs->digests, flavor_name, d);
+  }
+  return d;
+}
+
+/** Return the list of signatures of the <b>flavor_name</b>-flavored
+ * networkstatus according to the detached signatures document <b>sigs</b>,
+ * allocating a new common_digests_t as needed. */
+static smartlist_t *
+detached_get_signatures(ns_detached_signatures_t *sigs,
+                        const char *flavor_name)
+{
+  smartlist_t *sl = strmap_get(sigs->signatures, flavor_name);
+  if (!sl) {
+    sl = smartlist_new();
+    strmap_set(sigs->signatures, flavor_name, sl);
+  }
+  return sl;
+}
+
+/** Parse a detached v3 networkstatus signature document between <b>s</b> and
+ * <b>eos</b> and return the result.  Return -1 on failure. */
+ns_detached_signatures_t *
+networkstatus_parse_detached_signatures(const char *s, const char *eos)
+{
+  /* XXXX there is too much duplicate shared between this function and
+   * networkstatus_parse_vote_from_string(). */
+  directory_token_t *tok;
+  memarea_t *area = NULL;
+  common_digests_t *digests;
+
+  smartlist_t *tokens = smartlist_new();
+  ns_detached_signatures_t *sigs =
+    tor_malloc_zero(sizeof(ns_detached_signatures_t));
+  sigs->digests = strmap_new();
+  sigs->signatures = strmap_new();
+
+  if (!eos)
+    eos = s + strlen(s);
+
+  area = memarea_new();
+  if (tokenize_string(area,s, eos, tokens,
+                      networkstatus_detached_signature_token_table, 0)) {
+    log_warn(LD_DIR, "Error tokenizing detached networkstatus signatures");
+    goto err;
+  }
+
+  /* Grab all the digest-like tokens. */
+  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
+    const char *algname;
+    digest_algorithm_t alg;
+    const char *flavor;
+    const char *hexdigest;
+    size_t expected_length, digest_length;
+
+    tok = _tok;
+
+    if (tok->tp == K_CONSENSUS_DIGEST) {
+      algname = "sha1";
+      alg = DIGEST_SHA1;
+      flavor = "ns";
+      hexdigest = tok->args[0];
+    } else if (tok->tp == K_ADDITIONAL_DIGEST) {
+      int a = crypto_digest_algorithm_parse_name(tok->args[1]);
+      if (a<0) {
+        log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]);
+        continue;
+      }
+      alg = (digest_algorithm_t) a;
+      flavor = tok->args[0];
+      algname = tok->args[1];
+      hexdigest = tok->args[2];
+    } else {
+      continue;
+    }
+
+    digest_length = crypto_digest_algorithm_get_length(alg);
+    expected_length = digest_length * 2; /* hex encoding */
+
+    if (strlen(hexdigest) != expected_length) {
+      log_warn(LD_DIR, "Wrong length on consensus-digest in detached "
+               "networkstatus signatures");
+      goto err;
+    }
+    digests = detached_get_digests(sigs, flavor);
+    tor_assert(digests);
+    if (!tor_mem_is_zero(digests->d[alg], digest_length)) {
+      log_warn(LD_DIR, "Multiple digests for %s with %s on detached "
+               "signatures document", flavor, algname);
+      continue;
+    }
+    if (base16_decode(digests->d[alg], digest_length,
+                      hexdigest, strlen(hexdigest)) != (int) digest_length) {
+      log_warn(LD_DIR, "Bad encoding on consensus-digest in detached "
+               "networkstatus signatures");
+      goto err;
+    }
+  } SMARTLIST_FOREACH_END(_tok);
+
+  tok = find_by_keyword(tokens, K_VALID_AFTER);
+  if (parse_iso_time(tok->args[0], &sigs->valid_after)) {
+    log_warn(LD_DIR, "Bad valid-after in detached networkstatus signatures");
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_FRESH_UNTIL);
+  if (parse_iso_time(tok->args[0], &sigs->fresh_until)) {
+    log_warn(LD_DIR, "Bad fresh-until in detached networkstatus signatures");
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_VALID_UNTIL);
+  if (parse_iso_time(tok->args[0], &sigs->valid_until)) {
+    log_warn(LD_DIR, "Bad valid-until in detached networkstatus signatures");
+    goto err;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
+    const char *id_hexdigest;
+    const char *sk_hexdigest;
+    const char *algname;
+    const char *flavor;
+    digest_algorithm_t alg;
+
+    char id_digest[DIGEST_LEN];
+    char sk_digest[DIGEST_LEN];
+    smartlist_t *siglist;
+    document_signature_t *sig;
+    int is_duplicate;
+
+    tok = _tok;
+    if (tok->tp == K_DIRECTORY_SIGNATURE) {
+      tor_assert(tok->n_args >= 2);
+      flavor = "ns";
+      algname = "sha1";
+      id_hexdigest = tok->args[0];
+      sk_hexdigest = tok->args[1];
+    } else if (tok->tp == K_ADDITIONAL_SIGNATURE) {
+      tor_assert(tok->n_args >= 4);
+      flavor = tok->args[0];
+      algname = tok->args[1];
+      id_hexdigest = tok->args[2];
+      sk_hexdigest = tok->args[3];
+    } else {
+      continue;
+    }
+
+    {
+      int a = crypto_digest_algorithm_parse_name(algname);
+      if (a<0) {
+        log_warn(LD_DIR, "Unrecognized algorithm name %s", algname);
+        continue;
+      }
+      alg = (digest_algorithm_t) a;
+    }
+
+    if (!tok->object_type ||
+        strcmp(tok->object_type, "SIGNATURE") ||
+        tok->object_size < 128 || tok->object_size > 512) {
+      log_warn(LD_DIR, "Bad object type or length on directory-signature");
+      goto err;
+    }
+
+    if (strlen(id_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(id_digest, sizeof(id_digest),
+                      id_hexdigest, HEX_DIGEST_LEN) != sizeof(id_digest)) {
+      log_warn(LD_DIR, "Error decoding declared identity %s in "
+               "network-status vote.", escaped(id_hexdigest));
+      goto err;
+    }
+    if (strlen(sk_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(sk_digest, sizeof(sk_digest),
+                      sk_hexdigest, HEX_DIGEST_LEN) != sizeof(sk_digest)) {
+      log_warn(LD_DIR, "Error decoding declared signing key digest %s in "
+               "network-status vote.", escaped(sk_hexdigest));
+      goto err;
+    }
+
+    siglist = detached_get_signatures(sigs, flavor);
+    is_duplicate = 0;
+    SMARTLIST_FOREACH(siglist, document_signature_t *, dsig, {
+      if (dsig->alg == alg &&
+          tor_memeq(id_digest, dsig->identity_digest, DIGEST_LEN) &&
+          tor_memeq(sk_digest, dsig->signing_key_digest, DIGEST_LEN)) {
+        is_duplicate = 1;
+      }
+    });
+    if (is_duplicate) {
+      log_warn(LD_DIR, "Two signatures with identical keys and algorithm "
+               "found.");
+      continue;
+    }
+
+    sig = tor_malloc_zero(sizeof(document_signature_t));
+    sig->alg = alg;
+    memcpy(sig->identity_digest, id_digest, DIGEST_LEN);
+    memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN);
+    if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) {
+      tor_free(sig);
+      goto err;
+    }
+    sig->signature = tor_memdup(tok->object_body, tok->object_size);
+    sig->signature_len = (int) tok->object_size;
+
+    smartlist_add(siglist, sig);
+  } SMARTLIST_FOREACH_END(_tok);
+
+  goto done;
+ err:
+  ns_detached_signatures_free(sigs);
+  sigs = NULL;
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    DUMP_AREA(area, "detached signatures");
+    memarea_drop_all(area);
+  }
+  return sigs;
+}
+
+/** Parse the addr policy in the string <b>s</b> and return it.  If
+ * assume_action is nonnegative, then insert its action (ADDR_POLICY_ACCEPT or
+ * ADDR_POLICY_REJECT) for items that specify no action.
+ *
+ * Returns NULL on policy errors.
+ *
+ * Set *<b>malformed_list</b> to true if the entire policy list should be
+ * discarded. Otherwise, set it to false, and only this item should be ignored
+ * on error - the rest of the policy list can continue to be processed and
+ * used.
+ *
+ * The addr_policy_t returned by this function can have its address set to
+ * AF_UNSPEC for '*'.  Use policy_expand_unspec() to turn this into a pair
+ * of AF_INET and AF_INET6 items.
+ */
+MOCK_IMPL(addr_policy_t *,
+router_parse_addr_policy_item_from_string,(const char *s, int assume_action,
+                                           int *malformed_list))
+{
+  directory_token_t *tok = NULL;
+  const char *cp, *eos;
+  /* Longest possible policy is
+   * "accept6 [ffff:ffff:..255]/128:10000-65535",
+   * which contains a max-length IPv6 address, plus 26 characters.
+   * But note that there can be an arbitrary amount of space between the
+   * accept and the address:mask/port element.
+   * We don't need to multiply TOR_ADDR_BUF_LEN by 2, as there is only one
+   * IPv6 address. But making the buffer shorter might cause valid long lines,
+   * which parsed in previous versions, to fail to parse in new versions.
+   * (These lines would have to have excessive amounts of whitespace.) */
+  char line[TOR_ADDR_BUF_LEN*2 + 32];
+  addr_policy_t *r;
+  memarea_t *area = NULL;
+
+  tor_assert(malformed_list);
+  *malformed_list = 0;
+
+  s = eat_whitespace(s);
+  /* We can only do assume_action on []-quoted IPv6, as "a" (accept)
+   * and ":" (port separator) are ambiguous */
+  if ((*s == '*' || *s == '[' || TOR_ISDIGIT(*s)) && assume_action >= 0) {
+    if (tor_snprintf(line, sizeof(line), "%s %s",
+               assume_action == ADDR_POLICY_ACCEPT?"accept":"reject", s)<0) {
+      log_warn(LD_DIR, "Policy %s is too long.", escaped(s));
+      return NULL;
+    }
+    cp = line;
+    tor_strlower(line);
+  } else { /* assume an already well-formed address policy line */
+    cp = s;
+  }
+
+  eos = cp + strlen(cp);
+  area = memarea_new();
+  tok = get_next_token(area, &cp, eos, routerdesc_token_table);
+  if (tok->tp == ERR_) {
+    log_warn(LD_DIR, "Error reading address policy: %s", tok->error);
+    goto err;
+  }
+  if (tok->tp != K_ACCEPT && tok->tp != K_ACCEPT6 &&
+      tok->tp != K_REJECT && tok->tp != K_REJECT6) {
+    log_warn(LD_DIR, "Expected 'accept' or 'reject'.");
+    goto err;
+  }
+
+  /* Use the extended interpretation of accept/reject *,
+   * expanding it into an IPv4 wildcard and an IPv6 wildcard.
+   * Also permit *4 and *6 for IPv4 and IPv6 only wildcards. */
+  r = router_parse_addr_policy(tok, TAPMP_EXTENDED_STAR);
+  if (!r) {
+    goto err;
+  }
+
+  /* Ensure that accept6/reject6 fields are followed by IPv6 addresses.
+   * AF_UNSPEC addresses are only permitted on the accept/reject field type.
+   * Unlike descriptors, torrcs exit policy accept/reject can be followed by
+   * either an IPv4 or IPv6 address. */
+  if ((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) &&
+       tor_addr_family(&r->addr) != AF_INET6) {
+    /* This is a non-fatal error, just ignore this one entry. */
+    *malformed_list = 0;
+    log_warn(LD_DIR, "IPv4 address '%s' with accept6/reject6 field type in "
+             "exit policy. Ignoring, but continuing to parse rules. (Use "
+             "accept/reject with IPv4 addresses.)",
+             tok->n_args == 1 ? tok->args[0] : "");
+    addr_policy_free(r);
+    r = NULL;
+    goto done;
+  }
+
+  goto done;
+ err:
+  *malformed_list = 1;
+  r = NULL;
+ done:
+  token_clear(tok);
+  if (area) {
+    DUMP_AREA(area, "policy item");
+    memarea_drop_all(area);
+  }
+  return r;
+}
+
+/** Add an exit policy stored in the token <b>tok</b> to the router info in
+ * <b>router</b>.  Return 0 on success, -1 on failure. */
+static int
+router_add_exit_policy(routerinfo_t *router, directory_token_t *tok)
+{
+  addr_policy_t *newe;
+  /* Use the standard interpretation of accept/reject *, an IPv4 wildcard. */
+  newe = router_parse_addr_policy(tok, 0);
+  if (!newe)
+    return -1;
+  if (! router->exit_policy)
+    router->exit_policy = smartlist_new();
+
+  /* Ensure that in descriptors, accept/reject fields are followed by
+   * IPv4 addresses, and accept6/reject6 fields are followed by
+   * IPv6 addresses. Unlike torrcs, descriptor exit policies do not permit
+   * accept/reject followed by IPv6. */
+  if (((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) &&
+       tor_addr_family(&newe->addr) == AF_INET)
+      ||
+      ((tok->tp == K_ACCEPT || tok->tp == K_REJECT) &&
+       tor_addr_family(&newe->addr) == AF_INET6)) {
+    /* There's nothing the user can do about other relays' descriptors,
+     * so we don't provide usage advice here. */
+    log_warn(LD_DIR, "Mismatch between field type and address type in exit "
+             "policy '%s'. Discarding entire router descriptor.",
+             tok->n_args == 1 ? tok->args[0] : "");
+    addr_policy_free(newe);
+    return -1;
+  }
+
+  smartlist_add(router->exit_policy, newe);
+
+  return 0;
+}
+
+/** Given a K_ACCEPT[6] or K_REJECT[6] token and a router, create and return
+ * a new exit_policy_t corresponding to the token. If TAPMP_EXTENDED_STAR
+ * is set in fmt_flags, K_ACCEPT6 and K_REJECT6 tokens followed by *
+ * expand to IPv6-only policies, otherwise they expand to IPv4 and IPv6
+ * policies */
+static addr_policy_t *
+router_parse_addr_policy(directory_token_t *tok, unsigned fmt_flags)
+{
+  addr_policy_t newe;
+  char *arg;
+
+  tor_assert(tok->tp == K_REJECT || tok->tp == K_REJECT6 ||
+             tok->tp == K_ACCEPT || tok->tp == K_ACCEPT6);
+
+  if (tok->n_args != 1)
+    return NULL;
+  arg = tok->args[0];
+
+  if (!strcmpstart(arg,"private"))
+    return router_parse_addr_policy_private(tok);
+
+  memset(&newe, 0, sizeof(newe));
+
+  if (tok->tp == K_REJECT || tok->tp == K_REJECT6)
+    newe.policy_type = ADDR_POLICY_REJECT;
+  else
+    newe.policy_type = ADDR_POLICY_ACCEPT;
+
+  /* accept6/reject6 * produces an IPv6 wildcard address only.
+   * (accept/reject * produces rules for IPv4 and IPv6 wildcard addresses.) */
+  if ((fmt_flags & TAPMP_EXTENDED_STAR)
+      && (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6)) {
+    fmt_flags |= TAPMP_STAR_IPV6_ONLY;
+  }
+
+  if (tor_addr_parse_mask_ports(arg, fmt_flags, &newe.addr, &newe.maskbits,
+                                &newe.prt_min, &newe.prt_max) < 0) {
+    log_warn(LD_DIR,"Couldn't parse line %s. Dropping", escaped(arg));
+    return NULL;
+  }
+
+  return addr_policy_get_canonical_entry(&newe);
+}
+
+/** Parse an exit policy line of the format "accept[6]/reject[6] private:...".
+ * This didn't exist until Tor 0.1.1.15, so nobody should generate it in
+ * router descriptors until earlier versions are obsolete.
+ *
+ * accept/reject and accept6/reject6 private all produce rules for both
+ * IPv4 and IPv6 addresses.
+ */
+static addr_policy_t *
+router_parse_addr_policy_private(directory_token_t *tok)
+{
+  const char *arg;
+  uint16_t port_min, port_max;
+  addr_policy_t result;
+
+  arg = tok->args[0];
+  if (strcmpstart(arg, "private"))
+    return NULL;
+
+  arg += strlen("private");
+  arg = (char*) eat_whitespace(arg);
+  if (!arg || *arg != ':')
+    return NULL;
+
+  if (parse_port_range(arg+1, &port_min, &port_max)<0)
+    return NULL;
+
+  memset(&result, 0, sizeof(result));
+  if (tok->tp == K_REJECT || tok->tp == K_REJECT6)
+    result.policy_type = ADDR_POLICY_REJECT;
+  else
+    result.policy_type = ADDR_POLICY_ACCEPT;
+  result.is_private = 1;
+  result.prt_min = port_min;
+  result.prt_max = port_max;
+
+  if (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) {
+    log_warn(LD_GENERAL,
+             "'%s' expands into rules which apply to all private IPv4 and "
+             "IPv6 addresses. (Use accept/reject private:* for IPv4 and "
+             "IPv6.)", tok->n_args == 1 ? tok->args[0] : "");
+  }
+
+  return addr_policy_get_canonical_entry(&result);
+}
+
+/** Log and exit if <b>t</b> is malformed */
+void
+assert_addr_policy_ok(smartlist_t *lst)
+{
+  if (!lst) return;
+  SMARTLIST_FOREACH(lst, addr_policy_t *, t, {
+    tor_assert(t->policy_type == ADDR_POLICY_REJECT ||
+               t->policy_type == ADDR_POLICY_ACCEPT);
+    tor_assert(t->prt_min <= t->prt_max);
+  });
+}
+
+/** Return a newly allocated smartlist of all accept or reject tokens in
+ * <b>s</b>.
+ */
+static smartlist_t *
+find_all_exitpolicy(smartlist_t *s)
+{
+  smartlist_t *out = smartlist_new();
+  SMARTLIST_FOREACH(s, directory_token_t *, t,
+      if (t->tp == K_ACCEPT || t->tp == K_ACCEPT6 ||
+          t->tp == K_REJECT || t->tp == K_REJECT6)
+        smartlist_add(out,t));
+  return out;
+}
+
+/** Helper function for <b>router_get_hash_impl</b>: given <b>s</b>,
+ * <b>s_len</b>, <b>start_str</b>, <b>end_str</b>, and <b>end_c</b> with the
+ * same semantics as in that function, set *<b>start_out</b> (inclusive) and
+ * *<b>end_out</b> (exclusive) to the boundaries of the string to be hashed.
+ *
+ * Return 0 on success and -1 on failure.
+ */
+static int
+router_get_hash_impl_helper(const char *s, size_t s_len,
+                            const char *start_str,
+                            const char *end_str, char end_c,
+                            int log_severity,
+                            const char **start_out, const char **end_out)
+{
+  const char *start, *end;
+  start = tor_memstr(s, s_len, start_str);
+  if (!start) {
+    log_fn(log_severity,LD_DIR,
+           "couldn't find start of hashed material \"%s\"",start_str);
+    return -1;
+  }
+  if (start != s && *(start-1) != '\n') {
+    log_fn(log_severity,LD_DIR,
+             "first occurrence of \"%s\" is not at the start of a line",
+             start_str);
+    return -1;
+  }
+  end = tor_memstr(start+strlen(start_str),
+                   s_len - (start-s) - strlen(start_str), end_str);
+  if (!end) {
+    log_fn(log_severity,LD_DIR,
+           "couldn't find end of hashed material \"%s\"",end_str);
+    return -1;
+  }
+  end = memchr(end+strlen(end_str), end_c, s_len - (end-s) - strlen(end_str));
+  if (!end) {
+    log_fn(log_severity,LD_DIR,
+           "couldn't find EOL");
+    return -1;
+  }
+  ++end;
+
+  *start_out = start;
+  *end_out = end;
+  return 0;
+}
+
+/** Compute the digest of the substring of <b>s</b> taken from the first
+ * occurrence of <b>start_str</b> through the first instance of c after the
+ * first subsequent occurrence of <b>end_str</b>; store the 20-byte or 32-byte
+ * result in <b>digest</b>; return 0 on success.
+ *
+ * If no such substring exists, return -1.
+ */
+static int
+router_get_hash_impl(const char *s, size_t s_len, char *digest,
+                     const char *start_str,
+                     const char *end_str, char end_c,
+                     digest_algorithm_t alg)
+{
+  const char *start=NULL, *end=NULL;
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
+                                  &start,&end)<0)
+    return -1;
+
+  return router_compute_hash_final(digest, start, end-start, alg);
+}
+
+/** Compute the digest of the <b>len</b>-byte directory object at
+ * <b>start</b>, using <b>alg</b>. Store the result in <b>digest</b>, which
+ * must be long enough to hold it. */
+MOCK_IMPL(STATIC int,
+router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg))
+{
+  if (alg == DIGEST_SHA1) {
+    if (crypto_digest(digest, start, len) < 0) {
+      log_warn(LD_BUG,"couldn't compute digest");
+      return -1;
+    }
+  } else {
+    if (crypto_digest256(digest, start, len, alg) < 0) {
+      log_warn(LD_BUG,"couldn't compute digest");
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/** As router_get_hash_impl, but compute all hashes. */
+static int
+router_get_hashes_impl(const char *s, size_t s_len, common_digests_t *digests,
+                       const char *start_str,
+                       const char *end_str, char end_c)
+{
+  const char *start=NULL, *end=NULL;
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
+                                  &start,&end)<0)
+    return -1;
+
+  if (crypto_common_digests(digests, start, end-start)) {
+    log_warn(LD_BUG,"couldn't compute digests");
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Assuming that s starts with a microdesc, return the start of the
+ * *NEXT* one.  Return NULL on "not found." */
+static const char *
+find_start_of_next_microdesc(const char *s, const char *eos)
+{
+  int started_with_annotations;
+  s = eat_whitespace_eos(s, eos);
+  if (!s)
+    return NULL;
+
+#define CHECK_LENGTH() STMT_BEGIN \
+    if (s+32 > eos)               \
+      return NULL;                \
+  STMT_END
+
+#define NEXT_LINE() STMT_BEGIN            \
+    s = memchr(s, '\n', eos-s);           \
+    if (!s || s+1 >= eos)                 \
+      return NULL;                        \
+    s++;                                  \
+  STMT_END
+
+  CHECK_LENGTH();
+
+  started_with_annotations = (*s == '@');
+
+  if (started_with_annotations) {
+    /* Start by advancing to the first non-annotation line. */
+    while (*s == '@')
+      NEXT_LINE();
+  }
+  CHECK_LENGTH();
+
+  /* Now we should be pointed at an onion-key line.  If we are, then skip
+   * it. */
+  if (!strcmpstart(s, "onion-key"))
+    NEXT_LINE();
+
+  /* Okay, now we're pointed at the first line of the microdescriptor which is
+     not an annotation or onion-key.  The next line that _is_ an annotation or
+     onion-key is the start of the next microdescriptor. */
+  while (s+32 < eos) {
+    if (*s == '@' || !strcmpstart(s, "onion-key"))
+      return s;
+    NEXT_LINE();
+  }
+  return NULL;
+
+#undef CHECK_LENGTH
+#undef NEXT_LINE
+}
+
+/** Parse as many microdescriptors as are found from the string starting at
+ * <b>s</b> and ending at <b>eos</b>.  If allow_annotations is set, read any
+ * annotations we recognize and ignore ones we don't.
+ *
+ * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each
+ * descriptor in the body field of each microdesc_t.
+ *
+ * Return all newly parsed microdescriptors in a newly allocated
+ * smartlist_t. If <b>invalid_disgests_out</b> is provided, add a SHA256
+ * microdesc digest to it for every microdesc that we found to be badly
+ * formed. (This may cause duplicates) */
+smartlist_t *
+microdescs_parse_from_string(const char *s, const char *eos,
+                             int allow_annotations,
+                             saved_location_t where,
+                             smartlist_t *invalid_digests_out)
+{
+  smartlist_t *tokens;
+  smartlist_t *result;
+  microdesc_t *md = NULL;
+  memarea_t *area;
+  const char *start = s;
+  const char *start_of_next_microdesc;
+  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
+  const int copy_body = (where != SAVED_IN_CACHE);
+
+  directory_token_t *tok;
+
+  if (!eos)
+    eos = s + strlen(s);
+
+  s = eat_whitespace_eos(s, eos);
+  area = memarea_new();
+  result = smartlist_new();
+  tokens = smartlist_new();
+
+  while (s < eos) {
+    int okay = 0;
+
+    start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
+    if (!start_of_next_microdesc)
+      start_of_next_microdesc = eos;
+
+    md = tor_malloc_zero(sizeof(microdesc_t));
+    {
+      const char *cp = tor_memstr(s, start_of_next_microdesc-s,
+                                  "onion-key");
+      const int no_onion_key = (cp == NULL);
+      if (no_onion_key) {
+        cp = s; /* So that we have *some* junk to put in the body */
+      }
+
+      md->bodylen = start_of_next_microdesc - cp;
+      md->saved_location = where;
+      if (copy_body)
+        md->body = tor_memdup_nulterm(cp, md->bodylen);
+      else
+        md->body = (char*)cp;
+      md->off = cp - start;
+      crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
+      if (no_onion_key) {
+        log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor");
+        goto next;
+      }
+    }
+
+    if (tokenize_string(area, s, start_of_next_microdesc, tokens,
+                        microdesc_token_table, flags)) {
+      log_warn(LD_DIR, "Unparseable microdescriptor");
+      goto next;
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+      if (parse_iso_time(tok->args[0], &md->last_listed)) {
+        log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
+        goto next;
+      }
+    }
+
+    tok = find_by_keyword(tokens, K_ONION_KEY);
+    if (!crypto_pk_public_exponent_ok(tok->key)) {
+      log_warn(LD_DIR,
+               "Relay's onion key had invalid exponent.");
+      goto next;
+    }
+    md->onion_pkey = tok->key;
+    tok->key = NULL;
+
+    if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+      curve25519_public_key_t k;
+      tor_assert(tok->n_args >= 1);
+      if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+        log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
+        goto next;
+      }
+      md->onion_curve25519_pkey =
+        tor_memdup(&k, sizeof(curve25519_public_key_t));
+    }
+
+    smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
+    if (id_lines) {
+      SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
+        tor_assert(t->n_args >= 2);
+        if (!strcmp(t->args[0], "ed25519")) {
+          if (md->ed25519_identity_pkey) {
+            log_warn(LD_DIR, "Extra ed25519 key in microdesc");
+            smartlist_free(id_lines);
+            goto next;
+          }
+          ed25519_public_key_t k;
+          if (ed25519_public_from_base64(&k, t->args[1])<0) {
+            log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
+            smartlist_free(id_lines);
+            goto next;
+          }
+          md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
+        }
+      } SMARTLIST_FOREACH_END(t);
+      smartlist_free(id_lines);
+    }
+
+    {
+      smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+      if (a_lines) {
+        find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
+        smartlist_free(a_lines);
+      }
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
+      int i;
+      md->family = smartlist_new();
+      for (i=0;i<tok->n_args;++i) {
+        if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
+          log_warn(LD_DIR, "Illegal nickname %s in family line",
+                   escaped(tok->args[i]));
+          goto next;
+        }
+        smartlist_add_strdup(md->family, tok->args[i]);
+      }
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_P))) {
+      md->exit_policy = parse_short_policy(tok->args[0]);
+    }
+    if ((tok = find_opt_by_keyword(tokens, K_P6))) {
+      md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+    }
+
+    smartlist_add(result, md);
+    okay = 1;
+
+    md = NULL;
+  next:
+    if (! okay && invalid_digests_out) {
+      smartlist_add(invalid_digests_out,
+                    tor_memdup(md->digest, DIGEST256_LEN));
+    }
+    microdesc_free(md);
+    md = NULL;
+
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    memarea_clear(area);
+    smartlist_clear(tokens);
+    s = start_of_next_microdesc;
+  }
+
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+
+  return result;
+}
+
+/** Extract a Tor version from a <b>platform</b> line from a router
+ * descriptor, and place the result in <b>router_version</b>.
+ *
+ * Return 1 on success, -1 on parsing failure, and 0 if the
+ * platform line does not indicate some version of Tor.
+ *
+ * If <b>strict</b> is non-zero, finding any weird version components
+ * (like negative numbers) counts as a parsing failure.
+ */
+int
+tor_version_parse_platform(const char *platform,
+                           tor_version_t *router_version,
+                           int strict)
+{
+  char tmp[128];
+  char *s, *s2, *start;
+
+  if (strcmpstart(platform,"Tor ")) /* nonstandard Tor; say 0. */
+    return 0;
+
+  start = (char *)eat_whitespace(platform+3);
+  if (!*start) return -1;
+  s = (char *)find_whitespace(start); /* also finds '\0', which is fine */
+  s2 = (char*)eat_whitespace(s);
+  if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-"))
+    s = (char*)find_whitespace(s2);
+
+  if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */
+    return -1;
+  strlcpy(tmp, start, s-start+1);
+
+  if (tor_version_parse(tmp, router_version)<0) {
+    log_info(LD_DIR,"Router version '%s' unparseable.",tmp);
+    return -1;
+  }
+
+  if (strict) {
+    if (router_version->major < 0 ||
+        router_version->minor < 0 ||
+        router_version->micro < 0 ||
+        router_version->patchlevel < 0 ||
+        router_version->svn_revision < 0) {
+      return -1;
+    }
+  }
+
+  return 1;
+}
+
+/** Parse the Tor version of the platform string <b>platform</b>,
+ * and compare it to the version in <b>cutoff</b>. Return 1 if
+ * the router is at least as new as the cutoff, else return 0.
+ */
+int
+tor_version_as_new_as(const char *platform, const char *cutoff)
+{
+  tor_version_t cutoff_version, router_version;
+  int r;
+  tor_assert(platform);
+
+  if (tor_version_parse(cutoff, &cutoff_version)<0) {
+    log_warn(LD_BUG,"cutoff version '%s' unparseable.",cutoff);
+    return 0;
+  }
+
+  r = tor_version_parse_platform(platform, &router_version, 0);
+  if (r == 0) {
+    /* nonstandard Tor; be safe and say yes */
+    return 1;
+  } else if (r < 0) {
+    /* unparseable version; be safe and say yes. */
+    return 1;
+  }
+
+  /* Here's why we don't need to do any special handling for svn revisions:
+   * - If neither has an svn revision, we're fine.
+   * - If the router doesn't have an svn revision, we can't assume that it
+   *   is "at least" any svn revision, so we need to return 0.
+   * - If the target version doesn't have an svn revision, any svn revision
+   *   (or none at all) is good enough, so return 1.
+   * - If both target and router have an svn revision, we compare them.
+   */
+
+  return tor_version_compare(&router_version, &cutoff_version) >= 0;
+}
+
+/** Parse a tor version from <b>s</b>, and store the result in <b>out</b>.
+ * Return 0 on success, -1 on failure. */
+int
+tor_version_parse(const char *s, tor_version_t *out)
+{
+  char *eos=NULL;
+  const char *cp=NULL;
+  int ok = 1;
+  /* Format is:
+   *   "Tor " ? NUM dot NUM [ dot NUM [ ( pre | rc | dot ) NUM ] ] [ - tag ]
+   */
+  tor_assert(s);
+  tor_assert(out);
+
+  memset(out, 0, sizeof(tor_version_t));
+  out->status = VER_RELEASE;
+  if (!strcasecmpstart(s, "Tor "))
+    s += 4;
+
+  cp = s;
+
+#define NUMBER(m)                               \
+  do {                                          \
+    if (!cp || *cp < '0' || *cp > '9')          \
+      return -1;                                \
+    out->m = (int)tor_parse_uint64(cp, 10, 0, INT32_MAX, &ok, &eos);    \
+    if (!ok)                                    \
+      return -1;                                \
+    if (!eos || eos == cp)                      \
+      return -1;                                \
+    cp = eos;                                   \
+  } while (0)
+
+#define DOT()                                   \
+  do {                                          \
+    if (*cp != '.')                             \
+      return -1;                                \
+    ++cp;                                       \
+  } while (0)
+
+  NUMBER(major);
+  DOT();
+  NUMBER(minor);
+  if (*cp == 0)
+    return 0;
+  else if (*cp == '-')
+    goto status_tag;
+  DOT();
+  NUMBER(micro);
+
+  /* Get status */
+  if (*cp == 0) {
+    return 0;
+  } else if (*cp == '.') {
+    ++cp;
+  } else if (*cp == '-') {
+    goto status_tag;
+  } else if (0==strncmp(cp, "pre", 3)) {
+    out->status = VER_PRE;
+    cp += 3;
+  } else if (0==strncmp(cp, "rc", 2)) {
+    out->status = VER_RC;
+    cp += 2;
+  } else {
+    return -1;
+  }
+
+  NUMBER(patchlevel);
+
+ status_tag:
+  /* Get status tag. */
+  if (*cp == '-' || *cp == '.')
+    ++cp;
+  eos = (char*) find_whitespace(cp);
+  if (eos-cp >= (int)sizeof(out->status_tag))
+    strlcpy(out->status_tag, cp, sizeof(out->status_tag));
+  else {
+    memcpy(out->status_tag, cp, eos-cp);
+    out->status_tag[eos-cp] = 0;
+  }
+  cp = eat_whitespace(eos);
+
+  if (!strcmpstart(cp, "(r")) {
+    cp += 2;
+    out->svn_revision = (int) strtol(cp,&eos,10);
+  } else if (!strcmpstart(cp, "(git-")) {
+    char *close_paren = strchr(cp, ')');
+    int hexlen;
+    char digest[DIGEST_LEN];
+    if (! close_paren)
+      return -1;
+    cp += 5;
+    if (close_paren-cp > HEX_DIGEST_LEN)
+      return -1;
+    hexlen = (int)(close_paren-cp);
+    memwipe(digest, 0, sizeof(digest));
+    if ( hexlen == 0 || (hexlen % 2) == 1)
+      return -1;
+    if (base16_decode(digest, hexlen/2, cp, hexlen) != hexlen/2)
+      return -1;
+    memcpy(out->git_tag, digest, hexlen/2);
+    out->git_tag_len = hexlen/2;
+  }
+
+  return 0;
+#undef NUMBER
+#undef DOT
+}
+
+/** Compare two tor versions; Return <0 if a < b; 0 if a ==b, >0 if a >
+ * b. */
+int
+tor_version_compare(tor_version_t *a, tor_version_t *b)
+{
+  int i;
+  tor_assert(a);
+  tor_assert(b);
+
+  /* We take this approach to comparison to ensure the same (bogus!) behavior
+   * on all inputs as we would have seen before bug #21278 was fixed. The
+   * only important difference here is that this method doesn't cause
+   * a signed integer underflow.
+   */
+#define CMP(field) do {                               \
+    unsigned aval = (unsigned) a->field;              \
+    unsigned bval = (unsigned) b->field;              \
+    int result = (int) (aval - bval);                 \
+    if (result < 0)                                   \
+      return -1;                                      \
+    else if (result > 0)                              \
+      return 1;                                       \
+  } while (0)
+
+  CMP(major);
+  CMP(minor);
+  CMP(micro);
+  CMP(status);
+  CMP(patchlevel);
+  if ((i = strcmp(a->status_tag, b->status_tag)))
+     return i;
+  CMP(svn_revision);
+  CMP(git_tag_len);
+  if (a->git_tag_len)
+     return fast_memcmp(a->git_tag, b->git_tag, a->git_tag_len);
+  else
+     return 0;
+
+#undef CMP
+}
+
+/** Return true iff versions <b>a</b> and <b>b</b> belong to the same series.
+ */
+int
+tor_version_same_series(tor_version_t *a, tor_version_t *b)
+{
+  tor_assert(a);
+  tor_assert(b);
+  return ((a->major == b->major) &&
+          (a->minor == b->minor) &&
+          (a->micro == b->micro));
+}
+
+/** Helper: Given pointers to two strings describing tor versions, return -1
+ * if _a precedes _b, 1 if _b precedes _a, and 0 if they are equivalent.
+ * Used to sort a list of versions. */
+static int
+compare_tor_version_str_ptr_(const void **_a, const void **_b)
+{
+  const char *a = *_a, *b = *_b;
+  int ca, cb;
+  tor_version_t va, vb;
+  ca = tor_version_parse(a, &va);
+  cb = tor_version_parse(b, &vb);
+  /* If they both parse, compare them. */
+  if (!ca && !cb)
+    return tor_version_compare(&va,&vb);
+  /* If one parses, it comes first. */
+  if (!ca && cb)
+    return -1;
+  if (ca && !cb)
+    return 1;
+  /* If neither parses, compare strings.  Also, the directory server admin
+  ** needs to be smacked upside the head.  But Tor is tolerant and gentle. */
+  return strcmp(a,b);
+}
+
+/** Sort a list of string-representations of versions in ascending order. */
+void
+sort_version_list(smartlist_t *versions, int remove_duplicates)
+{
+  smartlist_sort(versions, compare_tor_version_str_ptr_);
+
+  if (remove_duplicates)
+    smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_);
+}
+
+/** Parse and validate the ASCII-encoded v2 descriptor in <b>desc</b>,
+ * write the parsed descriptor to the newly allocated *<b>parsed_out</b>, the
+ * binary descriptor ID of length DIGEST_LEN to <b>desc_id_out</b>, the
+ * encrypted introduction points to the newly allocated
+ * *<b>intro_points_encrypted_out</b>, their encrypted size to
+ * *<b>intro_points_encrypted_size_out</b>, the size of the encoded descriptor
+ * to *<b>encoded_size_out</b>, and a pointer to the possibly next
+ * descriptor to *<b>next_out</b>; return 0 for success (including validation)
+ * and -1 for failure.
+ *
+ * If <b>as_hsdir</b> is 1, we're parsing this as an HSDir, and we should
+ * be strict about time formats.
+ */
+int
+rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
+                                 char *desc_id_out,
+                                 char **intro_points_encrypted_out,
+                                 size_t *intro_points_encrypted_size_out,
+                                 size_t *encoded_size_out,
+                                 const char **next_out, const char *desc,
+                                 int as_hsdir)
+{
+  rend_service_descriptor_t *result =
+                            tor_malloc_zero(sizeof(rend_service_descriptor_t));
+  char desc_hash[DIGEST_LEN];
+  const char *eos;
+  smartlist_t *tokens = smartlist_new();
+  directory_token_t *tok;
+  char secret_id_part[DIGEST_LEN];
+  int i, version, num_ok=1;
+  smartlist_t *versions;
+  char public_key_hash[DIGEST_LEN];
+  char test_desc_id[DIGEST_LEN];
+  memarea_t *area = NULL;
+  const int strict_time_fmt = as_hsdir;
+
+  tor_assert(desc);
+  /* Check if desc starts correctly. */
+  if (strncmp(desc, "rendezvous-service-descriptor ",
+              strlen("rendezvous-service-descriptor "))) {
+    log_info(LD_REND, "Descriptor does not start correctly.");
+    goto err;
+  }
+  /* Compute descriptor hash for later validation. */
+  if (router_get_hash_impl(desc, strlen(desc), desc_hash,
+                           "rendezvous-service-descriptor ",
+                           "\nsignature", '\n', DIGEST_SHA1) < 0) {
+    log_warn(LD_REND, "Couldn't compute descriptor hash.");
+    goto err;
+  }
+  /* Determine end of string. */
+  eos = strstr(desc, "\nrendezvous-service-descriptor ");
+  if (!eos)
+    eos = desc + strlen(desc);
+  else
+    eos = eos + 1;
+  /* Check length. */
+  if (eos-desc > REND_DESC_MAX_SIZE) {
+    /* XXXX+ If we are parsing this descriptor as a server, this
+     * should be a protocol warning. */
+    log_warn(LD_REND, "Descriptor length is %d which exceeds "
+             "maximum rendezvous descriptor size of %d bytes.",
+             (int)(eos-desc), REND_DESC_MAX_SIZE);
+    goto err;
+  }
+  /* Tokenize descriptor. */
+  area = memarea_new();
+  if (tokenize_string(area, desc, eos, tokens, desc_token_table, 0)) {
+    log_warn(LD_REND, "Error tokenizing descriptor.");
+    goto err;
+  }
+  /* Set next to next descriptor, if available. */
+  *next_out = eos;
+  /* Set length of encoded descriptor. */
+  *encoded_size_out = eos - desc;
+  /* Check min allowed length of token list. */
+  if (smartlist_len(tokens) < 7) {
+    log_warn(LD_REND, "Impossibly short descriptor.");
+    goto err;
+  }
+  /* Parse base32-encoded descriptor ID. */
+  tok = find_by_keyword(tokens, R_RENDEZVOUS_SERVICE_DESCRIPTOR);
+  tor_assert(tok == smartlist_get(tokens, 0));
+  tor_assert(tok->n_args == 1);
+  if (!rend_valid_descriptor_id(tok->args[0])) {
+    log_warn(LD_REND, "Invalid descriptor ID: '%s'", tok->args[0]);
+    goto err;
+  }
+  if (base32_decode(desc_id_out, DIGEST_LEN,
+                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) < 0) {
+    log_warn(LD_REND, "Descriptor ID contains illegal characters: %s",
+             tok->args[0]);
+    goto err;
+  }
+  /* Parse descriptor version. */
+  tok = find_by_keyword(tokens, R_VERSION);
+  tor_assert(tok->n_args == 1);
+  result->version =
+    (int) tor_parse_long(tok->args[0], 10, 0, INT_MAX, &num_ok, NULL);
+  if (result->version != 2 || !num_ok) {
+    /* If it's <2, it shouldn't be under this format.  If the number
+     * is greater than 2, we bumped it because we broke backward
+     * compatibility.  See how version numbers in our other formats
+     * work. */
+    log_warn(LD_REND, "Unrecognized descriptor version: %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  /* Parse public key. */
+  tok = find_by_keyword(tokens, R_PERMANENT_KEY);
+  result->pk = tok->key;
+  tok->key = NULL; /* Prevent free */
+  /* Parse secret ID part. */
+  tok = find_by_keyword(tokens, R_SECRET_ID_PART);
+  tor_assert(tok->n_args == 1);
+  if (strlen(tok->args[0]) != REND_SECRET_ID_PART_LEN_BASE32 ||
+      strspn(tok->args[0], BASE32_CHARS) != REND_SECRET_ID_PART_LEN_BASE32) {
+    log_warn(LD_REND, "Invalid secret ID part: '%s'", tok->args[0]);
+    goto err;
+  }
+  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) < 0) {
+    log_warn(LD_REND, "Secret ID part contains illegal characters: %s",
+             tok->args[0]);
+    goto err;
+  }
+  /* Parse publication time -- up-to-date check is done when storing the
+   * descriptor. */
+  tok = find_by_keyword(tokens, R_PUBLICATION_TIME);
+  tor_assert(tok->n_args == 1);
+  if (parse_iso_time_(tok->args[0], &result->timestamp,
+                      strict_time_fmt, 0) < 0) {
+    log_warn(LD_REND, "Invalid publication time: '%s'", tok->args[0]);
+    goto err;
+  }
+  /* Parse protocol versions. */
+  tok = find_by_keyword(tokens, R_PROTOCOL_VERSIONS);
+  tor_assert(tok->n_args == 1);
+  versions = smartlist_new();
+  smartlist_split_string(versions, tok->args[0], ",",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  for (i = 0; i < smartlist_len(versions); i++) {
+    version = (int) tor_parse_long(smartlist_get(versions, i),
+                                   10, 0, INT_MAX, &num_ok, NULL);
+    if (!num_ok) /* It's a string; let's ignore it. */
+      continue;
+    if (version >= REND_PROTOCOL_VERSION_BITMASK_WIDTH)
+      /* Avoid undefined left-shift behaviour. */
+      continue;
+    result->protocols |= 1 << version;
+  }
+  SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp));
+  smartlist_free(versions);
+  /* Parse encrypted introduction points. Don't verify. */
+  tok = find_opt_by_keyword(tokens, R_INTRODUCTION_POINTS);
+  if (tok) {
+    if (strcmp(tok->object_type, "MESSAGE")) {
+      log_warn(LD_DIR, "Bad object type: introduction points should be of "
+               "type MESSAGE");
+      goto err;
+    }
+    *intro_points_encrypted_out = tor_memdup(tok->object_body,
+                                             tok->object_size);
+    *intro_points_encrypted_size_out = tok->object_size;
+  } else {
+    *intro_points_encrypted_out = NULL;
+    *intro_points_encrypted_size_out = 0;
+  }
+  /* Parse and verify signature. */
+  tok = find_by_keyword(tokens, R_SIGNATURE);
+  if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0,
+                            "v2 rendezvous service descriptor") < 0)
+    goto err;
+  /* Verify that descriptor ID belongs to public key and secret ID part. */
+  if (crypto_pk_get_digest(result->pk, public_key_hash) < 0) {
+    log_warn(LD_REND, "Unable to compute rend descriptor public key digest");
+    goto err;
+  }
+  rend_get_descriptor_id_bytes(test_desc_id, public_key_hash,
+                               secret_id_part);
+  if (tor_memneq(desc_id_out, test_desc_id, DIGEST_LEN)) {
+    log_warn(LD_REND, "Parsed descriptor ID does not match "
+             "computed descriptor ID.");
+    goto err;
+  }
+  goto done;
+ err:
+  rend_service_descriptor_free(result);
+  result = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area)
+    memarea_drop_all(area);
+  *parsed_out = result;
+  if (result)
+    return 0;
+  return -1;
+}
+
+/** Decrypt the encrypted introduction points in <b>ipos_encrypted</b> of
+ * length <b>ipos_encrypted_size</b> using <b>descriptor_cookie</b> and
+ * write the result to a newly allocated string that is pointed to by
+ * <b>ipos_decrypted</b> and its length to <b>ipos_decrypted_size</b>.
+ * Return 0 if decryption was successful and -1 otherwise. */
+int
+rend_decrypt_introduction_points(char **ipos_decrypted,
+                                 size_t *ipos_decrypted_size,
+                                 const char *descriptor_cookie,
+                                 const char *ipos_encrypted,
+                                 size_t ipos_encrypted_size)
+{
+  tor_assert(ipos_encrypted);
+  tor_assert(descriptor_cookie);
+  if (ipos_encrypted_size < 2) {
+    log_warn(LD_REND, "Size of encrypted introduction points is too "
+                      "small.");
+    return -1;
+  }
+  if (ipos_encrypted[0] == (int)REND_BASIC_AUTH) {
+    char iv[CIPHER_IV_LEN], client_id[REND_BASIC_AUTH_CLIENT_ID_LEN],
+         session_key[CIPHER_KEY_LEN], *dec;
+    int declen, client_blocks;
+    size_t pos = 0, len, client_entries_len;
+    crypto_digest_t *digest;
+    crypto_cipher_t *cipher;
+    client_blocks = (int) ipos_encrypted[1];
+    client_entries_len = client_blocks * REND_BASIC_AUTH_CLIENT_MULTIPLE *
+                         REND_BASIC_AUTH_CLIENT_ENTRY_LEN;
+    if (ipos_encrypted_size < 2 + client_entries_len + CIPHER_IV_LEN + 1) {
+      log_warn(LD_REND, "Size of encrypted introduction points is too "
+                        "small.");
+      return -1;
+    }
+    memcpy(iv, ipos_encrypted + 2 + client_entries_len, CIPHER_IV_LEN);
+    digest = crypto_digest_new();
+    crypto_digest_add_bytes(digest, descriptor_cookie, REND_DESC_COOKIE_LEN);
+    crypto_digest_add_bytes(digest, iv, CIPHER_IV_LEN);
+    crypto_digest_get_digest(digest, client_id,
+                             REND_BASIC_AUTH_CLIENT_ID_LEN);
+    crypto_digest_free(digest);
+    for (pos = 2; pos < 2 + client_entries_len;
+         pos += REND_BASIC_AUTH_CLIENT_ENTRY_LEN) {
+      if (tor_memeq(ipos_encrypted + pos, client_id,
+                  REND_BASIC_AUTH_CLIENT_ID_LEN)) {
+        /* Attempt to decrypt introduction points. */
+        cipher = crypto_cipher_new(descriptor_cookie);
+        if (crypto_cipher_decrypt(cipher, session_key, ipos_encrypted
+                                  + pos + REND_BASIC_AUTH_CLIENT_ID_LEN,
+                                  CIPHER_KEY_LEN) < 0) {
+          log_warn(LD_REND, "Could not decrypt session key for client.");
+          crypto_cipher_free(cipher);
+          return -1;
+        }
+        crypto_cipher_free(cipher);
+
+        len = ipos_encrypted_size - 2 - client_entries_len - CIPHER_IV_LEN;
+        dec = tor_malloc_zero(len + 1);
+        declen = crypto_cipher_decrypt_with_iv(session_key, dec, len,
+            ipos_encrypted + 2 + client_entries_len,
+            ipos_encrypted_size - 2 - client_entries_len);
+
+        if (declen < 0) {
+          log_warn(LD_REND, "Could not decrypt introduction point string.");
+          tor_free(dec);
+          return -1;
+        }
+        if (fast_memcmpstart(dec, declen, "introduction-point ")) {
+          log_warn(LD_REND, "Decrypted introduction points don't "
+                            "look like we could parse them.");
+          tor_free(dec);
+          continue;
+        }
+        *ipos_decrypted = dec;
+        *ipos_decrypted_size = declen;
+        return 0;
+      }
+    }
+    log_warn(LD_REND, "Could not decrypt introduction points. Please "
+             "check your authorization for this service!");
+    return -1;
+  } else if (ipos_encrypted[0] == (int)REND_STEALTH_AUTH) {
+    char *dec;
+    int declen;
+    if (ipos_encrypted_size < CIPHER_IV_LEN + 2) {
+      log_warn(LD_REND, "Size of encrypted introduction points is too "
+                        "small.");
+      return -1;
+    }
+    dec = tor_malloc_zero(ipos_encrypted_size - CIPHER_IV_LEN - 1 + 1);
+
+    declen = crypto_cipher_decrypt_with_iv(descriptor_cookie, dec,
+                                           ipos_encrypted_size -
+                                               CIPHER_IV_LEN - 1,
+                                           ipos_encrypted + 1,
+                                           ipos_encrypted_size - 1);
+
+    if (declen < 0) {
+      log_warn(LD_REND, "Decrypting introduction points failed!");
+      tor_free(dec);
+      return -1;
+    }
+    *ipos_decrypted = dec;
+    *ipos_decrypted_size = declen;
+    return 0;
+  } else {
+    log_warn(LD_REND, "Unknown authorization type number: %d",
+             ipos_encrypted[0]);
+    return -1;
+  }
+}
+
+/** Parse the encoded introduction points in <b>intro_points_encoded</b> of
+ * length <b>intro_points_encoded_size</b> and write the result to the
+ * descriptor in <b>parsed</b>; return the number of successfully parsed
+ * introduction points or -1 in case of a failure. */
+int
+rend_parse_introduction_points(rend_service_descriptor_t *parsed,
+                               const char *intro_points_encoded,
+                               size_t intro_points_encoded_size)
+{
+  const char *current_ipo, *end_of_intro_points;
+  smartlist_t *tokens = NULL;
+  directory_token_t *tok;
+  rend_intro_point_t *intro;
+  extend_info_t *info;
+  int result, num_ok=1;
+  memarea_t *area = NULL;
+  tor_assert(parsed);
+  /** Function may only be invoked once. */
+  tor_assert(!parsed->intro_nodes);
+  if (!intro_points_encoded || intro_points_encoded_size == 0) {
+    log_warn(LD_REND, "Empty or zero size introduction point list");
+    goto err;
+  }
+  /* Consider one intro point after the other. */
+  current_ipo = intro_points_encoded;
+  end_of_intro_points = intro_points_encoded + intro_points_encoded_size;
+  tokens = smartlist_new();
+  parsed->intro_nodes = smartlist_new();
+  area = memarea_new();
+
+  while (!fast_memcmpstart(current_ipo, end_of_intro_points-current_ipo,
+                      "introduction-point ")) {
+    /* Determine end of string. */
+    const char *eos = tor_memstr(current_ipo, end_of_intro_points-current_ipo,
+                                 "\nintroduction-point ");
+    if (!eos)
+      eos = end_of_intro_points;
+    else
+      eos = eos+1;
+    tor_assert(eos <= intro_points_encoded+intro_points_encoded_size);
+    /* Free tokens and clear token list. */
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_clear(tokens);
+    memarea_clear(area);
+    /* Tokenize string. */
+    if (tokenize_string(area, current_ipo, eos, tokens, ipo_token_table, 0)) {
+      log_warn(LD_REND, "Error tokenizing introduction point");
+      goto err;
+    }
+    /* Advance to next introduction point, if available. */
+    current_ipo = eos;
+    /* Check minimum allowed length of introduction point. */
+    if (smartlist_len(tokens) < 5) {
+      log_warn(LD_REND, "Impossibly short introduction point.");
+      goto err;
+    }
+    /* Allocate new intro point and extend info. */
+    intro = tor_malloc_zero(sizeof(rend_intro_point_t));
+    info = intro->extend_info = tor_malloc_zero(sizeof(extend_info_t));
+    /* Parse identifier. */
+    tok = find_by_keyword(tokens, R_IPO_IDENTIFIER);
+    if (base32_decode(info->identity_digest, DIGEST_LEN,
+                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) < 0) {
+      log_warn(LD_REND, "Identity digest contains illegal characters: %s",
+               tok->args[0]);
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    /* Write identifier to nickname. */
+    info->nickname[0] = '$';
+    base16_encode(info->nickname + 1, sizeof(info->nickname) - 1,
+                  info->identity_digest, DIGEST_LEN);
+    /* Parse IP address. */
+    tok = find_by_keyword(tokens, R_IPO_IP_ADDRESS);
+    if (tor_addr_parse(&info->addr, tok->args[0])<0) {
+      log_warn(LD_REND, "Could not parse introduction point address.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    if (tor_addr_family(&info->addr) != AF_INET) {
+      log_warn(LD_REND, "Introduction point address was not ipv4.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+
+    /* Parse onion port. */
+    tok = find_by_keyword(tokens, R_IPO_ONION_PORT);
+    info->port = (uint16_t) tor_parse_long(tok->args[0],10,1,65535,
+                                           &num_ok,NULL);
+    if (!info->port || !num_ok) {
+      log_warn(LD_REND, "Introduction point onion port %s is invalid",
+               escaped(tok->args[0]));
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    /* Parse onion key. */
+    tok = find_by_keyword(tokens, R_IPO_ONION_KEY);
+    if (!crypto_pk_public_exponent_ok(tok->key)) {
+      log_warn(LD_REND,
+               "Introduction point's onion key had invalid exponent.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    info->onion_key = tok->key;
+    tok->key = NULL; /* Prevent free */
+    /* Parse service key. */
+    tok = find_by_keyword(tokens, R_IPO_SERVICE_KEY);
+    if (!crypto_pk_public_exponent_ok(tok->key)) {
+      log_warn(LD_REND,
+               "Introduction point key had invalid exponent.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    intro->intro_key = tok->key;
+    tok->key = NULL; /* Prevent free */
+    /* Add extend info to list of introduction points. */
+    smartlist_add(parsed->intro_nodes, intro);
+  }
+  result = smartlist_len(parsed->intro_nodes);
+  goto done;
+
+ err:
+  result = -1;
+
+ done:
+  /* Free tokens and clear token list. */
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area)
+    memarea_drop_all(area);
+
+  return result;
+}
+
+/** Parse the content of a client_key file in <b>ckstr</b> and add
+ * rend_authorized_client_t's for each parsed client to
+ * <b>parsed_clients</b>. Return the number of parsed clients as result
+ * or -1 for failure. */
+int
+rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr)
+{
+  int result = -1;
+  smartlist_t *tokens;
+  directory_token_t *tok;
+  const char *current_entry = NULL;
+  memarea_t *area = NULL;
+  char *err_msg = NULL;
+  if (!ckstr || strlen(ckstr) == 0)
+    return -1;
+  tokens = smartlist_new();
+  /* Begin parsing with first entry, skipping comments or whitespace at the
+   * beginning. */
+  area = memarea_new();
+  current_entry = eat_whitespace(ckstr);
+  while (!strcmpstart(current_entry, "client-name ")) {
+    rend_authorized_client_t *parsed_entry;
+    /* Determine end of string. */
+    const char *eos = strstr(current_entry, "\nclient-name ");
+    if (!eos)
+      eos = current_entry + strlen(current_entry);
+    else
+      eos = eos + 1;
+    /* Free tokens and clear token list. */
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_clear(tokens);
+    memarea_clear(area);
+    /* Tokenize string. */
+    if (tokenize_string(area, current_entry, eos, tokens,
+                        client_keys_token_table, 0)) {
+      log_warn(LD_REND, "Error tokenizing client keys file.");
+      goto err;
+    }
+    /* Advance to next entry, if available. */
+    current_entry = eos;
+    /* Check minimum allowed length of token list. */
+    if (smartlist_len(tokens) < 2) {
+      log_warn(LD_REND, "Impossibly short client key entry.");
+      goto err;
+    }
+    /* Parse client name. */
+    tok = find_by_keyword(tokens, C_CLIENT_NAME);
+    tor_assert(tok == smartlist_get(tokens, 0));
+    tor_assert(tok->n_args == 1);
+
+    if (!rend_valid_client_name(tok->args[0])) {
+      log_warn(LD_CONFIG, "Illegal client name: %s. (Length must be "
+               "between 1 and %d, and valid characters are "
+               "[A-Za-z0-9+-_].)", tok->args[0], REND_CLIENTNAME_MAX_LEN);
+      goto err;
+    }
+    /* Check if client name is duplicate. */
+    if (strmap_get(parsed_clients, tok->args[0])) {
+      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains a "
+               "duplicate client name: '%s'. Ignoring.", tok->args[0]);
+      goto err;
+    }
+    parsed_entry = tor_malloc_zero(sizeof(rend_authorized_client_t));
+    parsed_entry->client_name = tor_strdup(tok->args[0]);
+    strmap_set(parsed_clients, parsed_entry->client_name, parsed_entry);
+    /* Parse client key. */
+    tok = find_opt_by_keyword(tokens, C_CLIENT_KEY);
+    if (tok) {
+      parsed_entry->client_key = tok->key;
+      tok->key = NULL; /* Prevent free */
+    }
+
+    /* Parse descriptor cookie. */
+    tok = find_by_keyword(tokens, C_DESCRIPTOR_COOKIE);
+    tor_assert(tok->n_args == 1);
+    if (rend_auth_decode_cookie(tok->args[0], parsed_entry->descriptor_cookie,
+                                NULL, &err_msg) < 0) {
+      tor_assert(err_msg);
+      log_warn(LD_REND, "%s", err_msg);
+      tor_free(err_msg);
+      goto err;
+    }
+  }
+  result = strmap_size(parsed_clients);
+  goto done;
+ err:
+  result = -1;
+ done:
+  /* Free tokens and clear token list. */
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area)
+    memarea_drop_all(area);
+  return result;
+}
+
+/** Called on startup; right now we just handle scanning the unparseable
+ * descriptor dumps, but hang anything else we might need to do in the
+ * future here as well.
+ */
+void
+routerparse_init(void)
+{
+  /*
+   * Check both if the sandbox is active and whether it's configured; no
+   * point in loading all that if we won't be able to use it after the
+   * sandbox becomes active.
+   */
+  if (!(sandbox_is_active() || get_options()->Sandbox)) {
+    dump_desc_init();
+  }
+}
+
+/** Clean up all data structures used by routerparse.c at exit */
+void
+routerparse_free_all(void)
+{
+  dump_desc_fifo_cleanup();
+}
diff --git a/src/feature/nodelist/routerparse.h b/src/feature/nodelist/routerparse.h
new file mode 100644
index 0000000000..87c2a75aa5
--- /dev/null
+++ b/src/feature/nodelist/routerparse.h
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerparse.h
+ * \brief Header file for routerparse.c.
+ **/
+
+#ifndef TOR_ROUTERPARSE_H
+#define TOR_ROUTERPARSE_H
+
+/** Possible statuses of a version of Tor, given opinions from the directory
+ * servers. */
+typedef enum version_status_t {
+  VS_RECOMMENDED=0, /**< This version is listed as recommended. */
+  VS_OLD=1, /**< This version is older than any recommended version. */
+  VS_NEW=2, /**< This version is newer than any recommended version. */
+  VS_NEW_IN_SERIES=3, /**< This version is newer than any recommended version
+                       * in its series, but later recommended versions exist.
+                       */
+  VS_UNRECOMMENDED=4, /**< This version is not recommended (general case). */
+  VS_EMPTY=5, /**< The version list was empty; no agreed-on versions. */
+  VS_UNKNOWN, /**< We have no idea. */
+} version_status_t;
+
+enum networkstatus_type_t;
+
+int router_get_router_hash(const char *s, size_t s_len, char *digest);
+int router_get_dir_hash(const char *s, char *digest);
+int router_get_networkstatus_v3_hashes(const char *s,
+                                       common_digests_t *digests);
+int router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                                  const char **start_out,
+                                                  const char **end_out);
+int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                               const char *s);
+int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest);
+#define DIROBJ_MAX_SIG_LEN 256
+char *router_get_dirobj_signature(const char *digest,
+                                  size_t digest_len,
+                                  const crypto_pk_t *private_key);
+int router_append_dirobj_signature(char *buf, size_t buf_len,
+                                   const char *digest,
+                                   size_t digest_len,
+                                   crypto_pk_t *private_key);
+int router_parse_list_from_string(const char **s, const char *eos,
+                                  smartlist_t *dest,
+                                  saved_location_t saved_location,
+                                  int is_extrainfo,
+                                  int allow_annotations,
+                                  const char *prepend_annotations,
+                                  smartlist_t *invalid_digests_out);
+
+routerinfo_t *router_parse_entry_from_string(const char *s, const char *end,
+                                             int cache_copy,
+                                             int allow_annotations,
+                                             const char *prepend_annotations,
+                                             int *can_dl_again_out);
+struct digest_ri_map_t;
+extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end,
+                             int cache_copy, struct digest_ri_map_t *routermap,
+                             int *can_dl_again_out);
+MOCK_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
+         (const char *s, int assume_action, int *malformed_list));
+version_status_t tor_version_is_obsolete(const char *myversion,
+                                         const char *versionlist);
+int tor_version_parse_platform(const char *platform,
+                               tor_version_t *version_out,
+                               int strict);
+int tor_version_as_new_as(const char *platform, const char *cutoff);
+int tor_version_parse(const char *s, tor_version_t *out);
+int tor_version_compare(tor_version_t *a, tor_version_t *b);
+int tor_version_same_series(tor_version_t *a, tor_version_t *b);
+void sort_version_list(smartlist_t *lst, int remove_duplicates);
+void assert_addr_policy_ok(smartlist_t *t);
+void dump_distinct_digest_count(int severity);
+
+int compare_vote_routerstatus_entries(const void **_a, const void **_b);
+int networkstatus_verify_bw_weights(networkstatus_t *ns, int);
+networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
+                                           const char **eos_out,
+                                           enum networkstatus_type_t ns_type);
+ns_detached_signatures_t *networkstatus_parse_detached_signatures(
+                                          const char *s, const char *eos);
+
+smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
+                                          int allow_annotations,
+                                          saved_location_t where,
+                                          smartlist_t *invalid_digests_out);
+
+authority_cert_t *authority_cert_parse_from_string(const char *s,
+                                                   const char **end_of_string);
+int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
+                                     char *desc_id_out,
+                                     char **intro_points_encrypted_out,
+                                     size_t *intro_points_encrypted_size_out,
+                                     size_t *encoded_size_out,
+                                     const char **next_out, const char *desc,
+                                     int as_hsdir);
+int rend_decrypt_introduction_points(char **ipos_decrypted,
+                                     size_t *ipos_decrypted_size,
+                                     const char *descriptor_cookie,
+                                     const char *ipos_encrypted,
+                                     size_t ipos_encrypted_size);
+int rend_parse_introduction_points(rend_service_descriptor_t *parsed,
+                                   const char *intro_points_encoded,
+                                   size_t intro_points_encoded_size);
+int rend_parse_client_keys(strmap_t *parsed_clients, const char *str);
+
+void routerparse_init(void);
+void routerparse_free_all(void);
+
+#ifdef ROUTERPARSE_PRIVATE
+/*
+ * One entry in the list of dumped descriptors; filename dumped to, length,
+ * SHA-256 and timestamp.
+ */
+
+typedef struct {
+  char *filename;
+  size_t len;
+  uint8_t digest_sha256[DIGEST256_LEN];
+  time_t when;
+} dumped_desc_t;
+
+EXTERN(uint64_t, len_descs_dumped)
+EXTERN(smartlist_t *, descs_dumped)
+STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
+                                            networkstatus_t *vote,
+                                            vote_routerstatus_t *vote_rs,
+                                            routerstatus_t *rs);
+MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
+    (const char *dirname, const char *f));
+STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
+STATIC void dump_desc_fifo_cleanup(void);
+struct memarea_t;
+STATIC routerstatus_t *routerstatus_parse_entry_from_string(
+                                     struct memarea_t *area,
+                                     const char **s, smartlist_t *tokens,
+                                     networkstatus_t *vote,
+                                     vote_routerstatus_t *vote_rs,
+                                     int consensus_method,
+                                     consensus_flavor_t flav);
+MOCK_DECL(STATIC void,dump_desc,(const char *desc, const char *type));
+MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg));
+MOCK_DECL(STATIC int, signed_digest_equals,
+          (const uint8_t *d1, const uint8_t *d2, size_t len));
+
+STATIC void summarize_protover_flags(protover_summary_flags_t *out,
+                                     const char *protocols,
+                                     const char *version);
+#endif /* defined(ROUTERPARSE_PRIVATE) */
+
+#define ED_DESC_SIGNATURE_PREFIX "Tor router descriptor signature v1"
+
+#endif /* !defined(TOR_ROUTERPARSE_H) */
diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c
new file mode 100644
index 0000000000..285ef9d821
--- /dev/null
+++ b/src/feature/nodelist/routerset.c
@@ -0,0 +1,463 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+n * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerset.c
+ *
+ * \brief Functions and structures to handle set-type selection of routers
+ *  by name, ID, address, etc.
+ *
+ * This module implements the routerset_t data structure, whose purpose
+ * is to specify a set of relays based on a list of their identities or
+ * properties.  Routersets can restrict relays by IP address mask,
+ * identity fingerprint, country codes, and nicknames (deprecated).
+ *
+ * Routersets are typically used for user-specified restrictions, and
+ * are created by invoking routerset_new and routerset_parse from
+ * config.c and confparse.c.  To use a routerset, invoke one of
+ * routerset_contains_...() functions , or use
+ * routerstatus_get_all_nodes() / routerstatus_subtract_nodes() to
+ * manipulate a smartlist of node_t pointers.
+ *
+ * Country-code restrictions are implemented in geoip.c.
+ */
+
+#define ROUTERSET_PRIVATE
+
+#include "or/or.h"
+#include "or/bridges.h"
+#include "or/geoip.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/router.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+
+#include "or/addr_policy_st.h"
+#include "or/extend_info_st.h"
+#include "or/node_st.h"
+#include "or/routerinfo_st.h"
+#include "or/routerstatus_st.h"
+
+/** Return a new empty routerset. */
+routerset_t *
+routerset_new(void)
+{
+  routerset_t *result = tor_malloc_zero(sizeof(routerset_t));
+  result->list = smartlist_new();
+  result->names = strmap_new();
+  result->digests = digestmap_new();
+  result->policies = smartlist_new();
+  result->country_names = smartlist_new();
+  return result;
+}
+
+/** If <b>c</b> is a country code in the form {cc}, return a newly allocated
+ * string holding the "cc" part.  Else, return NULL. */
+STATIC char *
+routerset_get_countryname(const char *c)
+{
+  char *country;
+
+  if (strlen(c) < 4 || c[0] !='{' || c[3] !='}')
+    return NULL;
+
+  country = tor_strndup(c+1, 2);
+  tor_strlower(country);
+  return country;
+}
+
+/** Update the routerset's <b>countries</b> bitarray_t. Called whenever
+ * the GeoIP IPv4 database is reloaded.
+ */
+void
+routerset_refresh_countries(routerset_t *target)
+{
+  int cc;
+  bitarray_free(target->countries);
+
+  if (!geoip_is_loaded(AF_INET)) {
+    target->countries = NULL;
+    target->n_countries = 0;
+    return;
+  }
+  target->n_countries = geoip_get_n_countries();
+  target->countries = bitarray_init_zero(target->n_countries);
+  SMARTLIST_FOREACH_BEGIN(target->country_names, const char *, country) {
+    cc = geoip_get_country(country);
+    if (cc >= 0) {
+      tor_assert(cc < target->n_countries);
+      bitarray_set(target->countries, cc);
+    } else {
+      log_warn(LD_CONFIG, "Country code '%s' is not recognized.",
+          country);
+    }
+  } SMARTLIST_FOREACH_END(country);
+}
+
+/** Parse the string <b>s</b> to create a set of routerset entries, and add
+ * them to <b>target</b>.  In log messages, refer to the string as
+ * <b>description</b>.  Return 0 on success, -1 on failure.
+ *
+ * Three kinds of elements are allowed in routersets: nicknames, IP address
+ * patterns, and fingerprints.  They may be surrounded by optional space, and
+ * must be separated by commas.
+ */
+int
+routerset_parse(routerset_t *target, const char *s, const char *description)
+{
+  int r = 0;
+  int added_countries = 0;
+  char *countryname;
+  smartlist_t *list = smartlist_new();
+  int malformed_list;
+  smartlist_split_string(list, s, ",",
+                         SPLIT_SKIP_SPACE | SPLIT_IGNORE_BLANK, 0);
+  SMARTLIST_FOREACH_BEGIN(list, char *, nick) {
+      addr_policy_t *p;
+      /* if it doesn't pass our validation, assume it's malformed */
+      malformed_list = 1;
+      if (is_legal_hexdigest(nick)) {
+        char d[DIGEST_LEN];
+        if (*nick == '$')
+          ++nick;
+        log_debug(LD_CONFIG, "Adding identity %s to %s", nick, description);
+        base16_decode(d, sizeof(d), nick, HEX_DIGEST_LEN);
+        digestmap_set(target->digests, d, (void*)1);
+      } else if (is_legal_nickname(nick)) {
+        log_debug(LD_CONFIG, "Adding nickname %s to %s", nick, description);
+        strmap_set_lc(target->names, nick, (void*)1);
+      } else if ((countryname = routerset_get_countryname(nick)) != NULL) {
+        log_debug(LD_CONFIG, "Adding country %s to %s", nick,
+                  description);
+        smartlist_add(target->country_names, countryname);
+        added_countries = 1;
+      } else if ((strchr(nick,'.') || strchr(nick, ':') ||  strchr(nick, '*'))
+                 && (p = router_parse_addr_policy_item_from_string(
+                                     nick, ADDR_POLICY_REJECT,
+                                     &malformed_list))) {
+        /* IPv4 addresses contain '.', IPv6 addresses contain ':',
+         * and wildcard addresses contain '*'. */
+        log_debug(LD_CONFIG, "Adding address %s to %s", nick, description);
+        smartlist_add(target->policies, p);
+      } else if (malformed_list) {
+        log_warn(LD_CONFIG, "Entry '%s' in %s is malformed. Discarding entire"
+                 " list.", nick, description);
+        r = -1;
+        tor_free(nick);
+        SMARTLIST_DEL_CURRENT(list, nick);
+      } else {
+        log_notice(LD_CONFIG, "Entry '%s' in %s is ignored. Using the"
+                   " remainder of the list.", nick, description);
+        tor_free(nick);
+        SMARTLIST_DEL_CURRENT(list, nick);
+      }
+  } SMARTLIST_FOREACH_END(nick);
+  policy_expand_unspec(&target->policies);
+  smartlist_add_all(target->list, list);
+  smartlist_free(list);
+  if (added_countries)
+    routerset_refresh_countries(target);
+  return r;
+}
+
+/** Add all members of the set <b>source</b> to <b>target</b>. */
+void
+routerset_union(routerset_t *target, const routerset_t *source)
+{
+  char *s;
+  tor_assert(target);
+  if (!source || !source->list)
+    return;
+  s = routerset_to_string(source);
+  routerset_parse(target, s, "other routerset");
+  tor_free(s);
+}
+
+/** Return true iff <b>set</b> lists only nicknames and digests, and includes
+ * no IP ranges or countries. */
+int
+routerset_is_list(const routerset_t *set)
+{
+  return smartlist_len(set->country_names) == 0 &&
+    smartlist_len(set->policies) == 0;
+}
+
+/** Return true iff we need a GeoIP IP-to-country database to make sense of
+ * <b>set</b>. */
+int
+routerset_needs_geoip(const routerset_t *set)
+{
+  return set && smartlist_len(set->country_names);
+}
+
+/** Return true iff there are no entries in <b>set</b>. */
+int
+routerset_is_empty(const routerset_t *set)
+{
+  return !set || smartlist_len(set->list) == 0;
+}
+
+/** Return the number of entries in <b>set</b>. This does NOT return a
+ * negative value. */
+int
+routerset_len(const routerset_t *set)
+{
+  if (!set) {
+    return 0;
+  }
+  return smartlist_len(set->list);
+}
+
+/** Helper.  Return true iff <b>set</b> contains a router based on the other
+ * provided fields.  Return higher values for more specific subentries: a
+ * single router is more specific than an address range of routers, which is
+ * more specific in turn than a country code.
+ *
+ * (If country is -1, then we take the country
+ * from addr.) */
+STATIC int
+routerset_contains(const routerset_t *set, const tor_addr_t *addr,
+                   uint16_t orport,
+                   const char *nickname, const char *id_digest,
+                   country_t country)
+{
+  if (!set || !set->list)
+    return 0;
+  if (nickname && strmap_get_lc(set->names, nickname))
+    return 4;
+  if (id_digest && digestmap_get(set->digests, id_digest))
+    return 4;
+  if (addr && compare_tor_addr_to_addr_policy(addr, orport, set->policies)
+      == ADDR_POLICY_REJECTED)
+    return 3;
+  if (set->countries) {
+    if (country < 0 && addr)
+      country = geoip_get_country_by_addr(addr);
+
+    if (country >= 0 && country < set->n_countries &&
+        bitarray_is_set(set->countries, country))
+      return 2;
+  }
+  return 0;
+}
+
+/** If *<b>setp</b> includes at least one country code, or if
+ * <b>only_some_cc_set</b> is 0, add the ?? and A1 country codes to
+ * *<b>setp</b>, creating it as needed.  Return true iff *<b>setp</b> changed.
+ */
+int
+routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set)
+{
+  routerset_t *set;
+  int add_unknown, add_a1;
+  if (only_if_some_cc_set) {
+    if (!*setp || smartlist_len((*setp)->country_names) == 0)
+      return 0;
+  }
+  if (!*setp)
+    *setp = routerset_new();
+
+  set = *setp;
+
+  add_unknown = ! smartlist_contains_string_case(set->country_names, "??") &&
+    geoip_get_country("??") >= 0;
+  add_a1 = ! smartlist_contains_string_case(set->country_names, "a1") &&
+    geoip_get_country("A1") >= 0;
+
+  if (add_unknown) {
+    smartlist_add_strdup(set->country_names, "??");
+    smartlist_add_strdup(set->list, "{??}");
+  }
+  if (add_a1) {
+    smartlist_add_strdup(set->country_names, "a1");
+    smartlist_add_strdup(set->list, "{a1}");
+  }
+
+  if (add_unknown || add_a1) {
+    routerset_refresh_countries(set);
+    return 1;
+  }
+  return 0;
+}
+
+/** Return true iff we can tell that <b>ei</b> is a member of <b>set</b>. */
+int
+routerset_contains_extendinfo(const routerset_t *set, const extend_info_t *ei)
+{
+  return routerset_contains(set,
+                            &ei->addr,
+                            ei->port,
+                            ei->nickname,
+                            ei->identity_digest,
+                            -1 /*country*/);
+}
+
+/** Return true iff <b>ri</b> is in <b>set</b>.  If country is <b>-1</b>, we
+ * look up the country. */
+int
+routerset_contains_router(const routerset_t *set, const routerinfo_t *ri,
+                          country_t country)
+{
+  tor_addr_t addr;
+  tor_addr_from_ipv4h(&addr, ri->addr);
+  return routerset_contains(set,
+                            &addr,
+                            ri->or_port,
+                            ri->nickname,
+                            ri->cache_info.identity_digest,
+                            country);
+}
+
+/** Return true iff <b>rs</b> is in <b>set</b>.  If country is <b>-1</b>, we
+ * look up the country. */
+int
+routerset_contains_routerstatus(const routerset_t *set,
+                                const routerstatus_t *rs,
+                                country_t country)
+{
+  tor_addr_t addr;
+  tor_addr_from_ipv4h(&addr, rs->addr);
+  return routerset_contains(set,
+                            &addr,
+                            rs->or_port,
+                            rs->nickname,
+                            rs->identity_digest,
+                            country);
+}
+
+/** Return true iff <b>node</b> is in <b>set</b>. */
+int
+routerset_contains_node(const routerset_t *set, const node_t *node)
+{
+  if (node->rs)
+    return routerset_contains_routerstatus(set, node->rs, node->country);
+  else if (node->ri)
+    return routerset_contains_router(set, node->ri, node->country);
+  else
+    return 0;
+}
+
+/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */
+int
+routerset_contains_bridge(const routerset_t *set, const bridge_info_t *bridge)
+{
+  const char *id = (const char*)bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+
+  tor_assert(addrport);
+  return routerset_contains(set, &addrport->addr, addrport->port,
+                            NULL, id, -1);
+}
+
+/** Add every known node_t that is a member of <b>routerset</b> to
+ * <b>out</b>, but never add any that are part of <b>excludeset</b>.
+ * If <b>running_only</b>, only add the running ones. */
+void
+routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
+                        const routerset_t *excludeset, int running_only)
+{
+  tor_assert(out);
+  if (!routerset || !routerset->list)
+    return;
+
+  if (routerset_is_list(routerset)) {
+    /* No routers are specified by type; all are given by name or digest.
+     * we can do a lookup in O(len(routerset)). */
+    SMARTLIST_FOREACH(routerset->list, const char *, name, {
+        const node_t *node = node_get_by_nickname(name, 0);
+        if (node) {
+          if (!running_only || node->is_running)
+            if (!routerset_contains_node(excludeset, node))
+              smartlist_add(out, (void*)node);
+        }
+    });
+  } else {
+    /* We need to iterate over the routerlist to get all the ones of the
+     * right kind. */
+    smartlist_t *nodes = nodelist_get_list();
+    SMARTLIST_FOREACH(nodes, const node_t *, node, {
+        if (running_only && !node->is_running)
+          continue;
+        if (routerset_contains_node(routerset, node) &&
+            !routerset_contains_node(excludeset, node))
+          smartlist_add(out, (void*)node);
+    });
+  }
+}
+
+/** Remove every node_t from <b>lst</b> that is in <b>routerset</b>. */
+void
+routerset_subtract_nodes(smartlist_t *lst, const routerset_t *routerset)
+{
+  tor_assert(lst);
+  if (!routerset)
+    return;
+  SMARTLIST_FOREACH(lst, const node_t *, node, {
+      if (routerset_contains_node(routerset, node)) {
+        //log_debug(LD_DIR, "Subtracting %s",r->nickname);
+        SMARTLIST_DEL_CURRENT(lst, node);
+      }
+    });
+}
+
+/** Return a new string that when parsed by routerset_parse_string() will
+ * yield <b>set</b>. */
+char *
+routerset_to_string(const routerset_t *set)
+{
+  if (!set || !set->list)
+    return tor_strdup("");
+  return smartlist_join_strings(set->list, ",", 0, NULL);
+}
+
+/** Helper: return true iff old and new are both NULL, or both non-NULL
+ * equal routersets. */
+int
+routerset_equal(const routerset_t *old, const routerset_t *new)
+{
+  if (routerset_is_empty(old) && routerset_is_empty(new)) {
+    /* Two empty sets are equal */
+    return 1;
+  } else if (routerset_is_empty(old) || routerset_is_empty(new)) {
+    /* An empty set is equal to nothing else. */
+    return 0;
+  }
+  tor_assert(old != NULL);
+  tor_assert(new != NULL);
+
+  if (smartlist_len(old->list) != smartlist_len(new->list))
+    return 0;
+
+  SMARTLIST_FOREACH(old->list, const char *, cp1, {
+    const char *cp2 = smartlist_get(new->list, cp1_sl_idx);
+    if (strcmp(cp1, cp2))
+      return 0;
+  });
+
+  return 1;
+}
+
+/** Free all storage held in <b>routerset</b>. */
+void
+routerset_free_(routerset_t *routerset)
+{
+  if (!routerset)
+    return;
+
+  SMARTLIST_FOREACH(routerset->list, char *, cp, tor_free(cp));
+  smartlist_free(routerset->list);
+  SMARTLIST_FOREACH(routerset->policies, addr_policy_t *, p,
+                    addr_policy_free(p));
+  smartlist_free(routerset->policies);
+  SMARTLIST_FOREACH(routerset->country_names, char *, cp, tor_free(cp));
+  smartlist_free(routerset->country_names);
+
+  strmap_free(routerset->names, NULL);
+  digestmap_free(routerset->digests, NULL);
+  bitarray_free(routerset->countries);
+  tor_free(routerset);
+}
diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h
new file mode 100644
index 0000000000..8a13ca042a
--- /dev/null
+++ b/src/feature/nodelist/routerset.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerlist.h
+ * \brief Header file for routerset.c
+ **/
+
+#ifndef TOR_ROUTERSET_H
+#define TOR_ROUTERSET_H
+
+routerset_t *routerset_new(void);
+void routerset_refresh_countries(routerset_t *rs);
+int routerset_parse(routerset_t *target, const char *s,
+                    const char *description);
+void routerset_union(routerset_t *target, const routerset_t *source);
+int routerset_is_list(const routerset_t *set);
+int routerset_needs_geoip(const routerset_t *set);
+int routerset_is_empty(const routerset_t *set);
+int routerset_contains_router(const routerset_t *set, const routerinfo_t *ri,
+                              country_t country);
+int routerset_contains_routerstatus(const routerset_t *set,
+                                    const routerstatus_t *rs,
+                                    country_t country);
+int routerset_contains_extendinfo(const routerset_t *set,
+                                  const extend_info_t *ei);
+struct bridge_info_t;
+int routerset_contains_bridge(const routerset_t *set,
+                              const struct bridge_info_t *bridge);
+int routerset_contains_node(const routerset_t *set, const node_t *node);
+
+void routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
+                             const routerset_t *excludeset,
+                             int running_only);
+int routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set);
+void routerset_subtract_nodes(smartlist_t *out,
+                                const routerset_t *routerset);
+
+char *routerset_to_string(const routerset_t *routerset);
+int routerset_equal(const routerset_t *old, const routerset_t *new);
+void routerset_free_(routerset_t *routerset);
+#define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs))
+int routerset_len(const routerset_t *set);
+
+#ifdef ROUTERSET_PRIVATE
+#include "lib/container/bitarray.h"
+
+STATIC char * routerset_get_countryname(const char *c);
+STATIC int routerset_contains(const routerset_t *set, const tor_addr_t *addr,
+                   uint16_t orport,
+                   const char *nickname, const char *id_digest,
+                   country_t country);
+
+/** A routerset specifies constraints on a set of possible routerinfos, based
+ * on their names, identities, or addresses.  It is optimized for determining
+ * whether a router is a member or not, in O(1+P) time, where P is the number
+ * of address policy constraints. */
+struct routerset_t {
+  /** A list of strings for the elements of the policy.  Each string is either
+   * a nickname, a hexadecimal identity fingerprint, or an address policy.  A
+   * router belongs to the set if its nickname OR its identity OR its address
+   * matches an entry here. */
+  smartlist_t *list;
+  /** A map from lowercase nicknames of routers in the set to (void*)1 */
+  strmap_t *names;
+  /** A map from identity digests routers in the set to (void*)1 */
+  digestmap_t *digests;
+  /** An address policy for routers in the set.  For implementation reasons,
+   * a router belongs to the set if it is _rejected_ by this policy. */
+  smartlist_t *policies;
+
+  /** A human-readable description of what this routerset is for.  Used in
+   * log messages. */
+  char *description;
+
+  /** A list of the country codes in this set. */
+  smartlist_t *country_names;
+  /** Total number of countries we knew about when we built <b>countries</b>.*/
+  int n_countries;
+  /** Bit array mapping the return value of geoip_get_country() to 1 iff the
+   * country is a member of this routerset.  Note that we MUST call
+   * routerset_refresh_countries() whenever the geoip country list is
+   * reloaded. */
+  bitarray_t *countries;
+};
+#endif /* defined(ROUTERSET_PRIVATE) */
+#endif /* !defined(TOR_ROUTERSET_H) */
diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h
new file mode 100644
index 0000000000..3de4a40ae4
--- /dev/null
+++ b/src/feature/nodelist/routerstatus_st.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ROUTERSTATUS_ST_H
+#define ROUTERSTATUS_ST_H
+
+#include "or/download_status_st.h"
+
+/** Contents of a single router entry in a network status object.
+ */
+struct routerstatus_t {
+  time_t published_on; /**< When was this router published? */
+  char nickname[MAX_NICKNAME_LEN+1]; /**< The nickname this router says it
+                                      * has. */
+  char identity_digest[DIGEST_LEN]; /**< Digest of the router's identity
+                                     * key. */
+  /** Digest of the router's most recent descriptor or microdescriptor.
+   * If it's a descriptor, we only use the first DIGEST_LEN bytes. */
+  char descriptor_digest[DIGEST256_LEN];
+  uint32_t addr; /**< IPv4 address for this router, in host order. */
+  uint16_t or_port; /**< IPv4 OR port for this router. */
+  uint16_t dir_port; /**< Directory port for this router. */
+  tor_addr_t ipv6_addr; /**< IPv6 address for this router. */
+  uint16_t ipv6_orport; /**< IPv6 OR port for this router. */
+  unsigned int is_authority:1; /**< True iff this router is an authority. */
+  unsigned int is_exit:1; /**< True iff this router is a good exit. */
+  unsigned int is_stable:1; /**< True iff this router stays up a long time. */
+  unsigned int is_fast:1; /**< True iff this router has good bandwidth. */
+  /** True iff this router is called 'running' in the consensus. We give it
+   * this funny name so that we don't accidentally use this bit as a view of
+   * whether we think the router is *currently* running.  If that's what you
+   * want to know, look at is_running in node_t. */
+  unsigned int is_flagged_running:1;
+  unsigned int is_named:1; /**< True iff "nickname" belongs to this router. */
+  unsigned int is_unnamed:1; /**< True iff "nickname" belongs to another
+                              * router. */
+  unsigned int is_valid:1; /**< True iff this router isn't invalid. */
+  unsigned int is_possible_guard:1; /**< True iff this router would be a good
+                                     * choice as an entry guard. */
+  unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for
+                               * an exit node. */
+  unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden
+                             * service directory. */
+  unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
+                             * or it claims to accept tunnelled dir requests.
+                             */
+
+  unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
+  unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
+  unsigned int bw_is_unmeasured:1; /**< This is a consensus entry, with
+                                    * the Unmeasured flag set. */
+
+  /** Flags to summarize the protocol versions for this routerstatus_t. */
+  protover_summary_flags_t pv;
+
+  uint32_t bandwidth_kb; /**< Bandwidth (capacity) of the router as reported in
+                       * the vote/consensus, in kilobytes/sec. */
+
+  /** The consensus has guardfraction information for this router. */
+  unsigned int has_guardfraction:1;
+  /** The guardfraction value of this router. */
+  uint32_t guardfraction_percentage;
+
+  char *exitsummary; /**< exit policy summary -
+                      * XXX weasel: this probably should not stay a string. */
+
+  /* ---- The fields below aren't derived from the networkstatus; they
+   * hold local information only. */
+
+  time_t last_dir_503_at; /**< When did this router last tell us that it
+                           * was too busy to serve directory info? */
+  download_status_t dl_status;
+
+};
+
+#endif
+
diff --git a/src/feature/nodelist/signed_descriptor_st.h b/src/feature/nodelist/signed_descriptor_st.h
new file mode 100644
index 0000000000..90cd4a2703
--- /dev/null
+++ b/src/feature/nodelist/signed_descriptor_st.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef SIGNED_DESCRIPTOR_ST_H
+#define SIGNED_DESCRIPTOR_ST_H
+
+#include "or/download_status_st.h"
+
+/** Information need to cache an onion router's descriptor. */
+struct signed_descriptor_t {
+  /** Pointer to the raw server descriptor, preceded by annotations.  Not
+   * necessarily NUL-terminated.  If saved_location is SAVED_IN_CACHE, this
+   * pointer is null. */
+  char *signed_descriptor_body;
+  /** Length of the annotations preceding the server descriptor. */
+  size_t annotations_len;
+  /** Length of the server descriptor. */
+  size_t signed_descriptor_len;
+  /** Digest of the server descriptor, computed as specified in
+   * dir-spec.txt. */
+  char signed_descriptor_digest[DIGEST_LEN];
+  /** Identity digest of the router. */
+  char identity_digest[DIGEST_LEN];
+  /** Declared publication time of the descriptor. */
+  time_t published_on;
+  /** For routerdescs only: digest of the corresponding extrainfo. */
+  char extra_info_digest[DIGEST_LEN];
+  /** For routerdescs only: A SHA256-digest of the extrainfo (if any) */
+  char extra_info_digest256[DIGEST256_LEN];
+  /** Certificate for ed25519 signing key. */
+  struct tor_cert_st *signing_key_cert;
+  /** For routerdescs only: Status of downloading the corresponding
+   * extrainfo. */
+  download_status_t ei_dl_status;
+  /** Where is the descriptor saved? */
+  saved_location_t saved_location;
+  /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of
+   * this descriptor in the corresponding file. */
+  off_t saved_offset;
+  /** What position is this descriptor within routerlist->routers or
+   * routerlist->old_routers? -1 for none. */
+  int routerlist_index;
+  /** The valid-until time of the most recent consensus that listed this
+   * descriptor.  0 for "never listed in a consensus, so far as we know." */
+  time_t last_listed_as_valid_until;
+  /* If true, we do not ever try to save this object in the cache. */
+  unsigned int do_not_cache : 1;
+  /* If true, this item is meant to represent an extrainfo. */
+  unsigned int is_extrainfo : 1;
+  /* If true, we got an extrainfo for this item, and the digest was right,
+   * but it was incompatible. */
+  unsigned int extrainfo_is_bogus : 1;
+  /* If true, we are willing to transmit this item unencrypted. */
+  unsigned int send_unencrypted : 1;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c
new file mode 100644
index 0000000000..39c6605c65
--- /dev/null
+++ b/src/feature/nodelist/torcert.c
@@ -0,0 +1,725 @@
+/* Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torcert.c
+ *
+ * \brief Implementation for ed25519-signed certificates as used in the Tor
+ * protocol.
+ *
+ * This certificate format is designed to be simple and compact; it's
+ * documented in tor-spec.txt in the torspec.git repository.  All of the
+ * certificates in this format are signed with an Ed25519 key; the
+ * contents themselves may be another Ed25519 key, a digest of a
+ * RSA key, or some other material.
+ *
+ * In this module there is also support for a crooss-certification of
+ * Ed25519 identities using (older) RSA1024 identities.
+ *
+ * Tor uses other types of certificate too, beyond those described in this
+ * module. Notably, our use of TLS requires us to touch X.509 certificates,
+ * even though sensible people would stay away from those. Our X.509
+ * certificates are represented with tor_x509_cert_t, and implemented in
+ * tortls.c.  We also have a separate certificate type that authorities
+ * use to authenticate their RSA signing keys with their RSA identity keys:
+ * that one is authority_cert_t, and it's mostly handled in routerlist.c.
+ */
+
+#include "or/or.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/torcert.h"
+#include "trunnel/ed25519_cert.h"
+#include "lib/log/torlog.h"
+#include "trunnel/link_handshake.h"
+#include "lib/tls/tortls.h"
+
+#include "or/or_handshake_certs_st.h"
+
+/** Helper for tor_cert_create(): signs any 32 bytes, not just an ed25519
+ * key.
+ */
+static tor_cert_t *
+tor_cert_sign_impl(const ed25519_keypair_t *signing_key,
+                      uint8_t cert_type,
+                      uint8_t signed_key_type,
+                      const uint8_t signed_key_info[32],
+                      time_t now, time_t lifetime,
+                      uint32_t flags)
+{
+  tor_cert_t *torcert = NULL;
+
+  ed25519_cert_t *cert = ed25519_cert_new();
+  cert->cert_type = cert_type;
+  cert->exp_field = (uint32_t) CEIL_DIV(now + lifetime, 3600);
+  cert->cert_key_type = signed_key_type;
+  memcpy(cert->certified_key, signed_key_info, 32);
+
+  if (flags & CERT_FLAG_INCLUDE_SIGNING_KEY) {
+    ed25519_cert_extension_t *ext = ed25519_cert_extension_new();
+    ext->ext_type = CERTEXT_SIGNED_WITH_KEY;
+    memcpy(ext->un_signing_key, signing_key->pubkey.pubkey, 32);
+    ed25519_cert_add_ext(cert, ext);
+    ++cert->n_extensions;
+  }
+
+  const ssize_t alloc_len = ed25519_cert_encoded_len(cert);
+  tor_assert(alloc_len > 0);
+  uint8_t *encoded = tor_malloc(alloc_len);
+  const ssize_t real_len = ed25519_cert_encode(encoded, alloc_len, cert);
+  if (real_len < 0)
+    goto err;
+  tor_assert(real_len == alloc_len);
+  tor_assert(real_len > ED25519_SIG_LEN);
+  uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN);
+  tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN));
+
+  ed25519_signature_t signature;
+  if (ed25519_sign(&signature, encoded,
+                   real_len-ED25519_SIG_LEN, signing_key)<0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Can't sign certificate");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+  memcpy(sig, signature.sig, ED25519_SIG_LEN);
+
+  torcert = tor_cert_parse(encoded, real_len);
+  if (! torcert) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Generated a certificate we cannot parse");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (tor_cert_checksig(torcert, &signing_key->pubkey, now) < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Generated a certificate whose signature we can't "
+             "check: %s", tor_cert_describe_signature_status(torcert));
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  tor_free(encoded);
+
+  goto done;
+
+ /* LCOV_EXCL_START */
+ err:
+  tor_cert_free(torcert);
+  torcert = NULL;
+ /* LCOV_EXCL_STOP */
+
+ done:
+  ed25519_cert_free(cert);
+  tor_free(encoded);
+  return torcert;
+}
+
+/**
+ * Create and return a new new certificate of type <b>cert_type</b> to
+ * authenticate <b>signed_key</b> using the key <b>signing_key</b>.  The
+ * certificate should remain valid for at least <b>lifetime</b> seconds after
+ * <b>now</b>.
+ *
+ * If CERT_FLAG_INCLUDE_SIGNING_KEY is set in <b>flags</b>, embed
+ * the public part of <b>signing_key</b> in the certificate.
+ */
+tor_cert_t *
+tor_cert_create(const ed25519_keypair_t *signing_key,
+                uint8_t cert_type,
+                const ed25519_public_key_t *signed_key,
+                time_t now, time_t lifetime,
+                uint32_t flags)
+{
+  return tor_cert_sign_impl(signing_key, cert_type,
+                            SIGNED_KEY_TYPE_ED25519, signed_key->pubkey,
+                            now, lifetime, flags);
+}
+
+/** Release all storage held for <b>cert</b>. */
+void
+tor_cert_free_(tor_cert_t *cert)
+{
+  if (! cert)
+    return;
+
+  if (cert->encoded)
+    memwipe(cert->encoded, 0, cert->encoded_len);
+  tor_free(cert->encoded);
+
+  memwipe(cert, 0, sizeof(tor_cert_t));
+  tor_free(cert);
+}
+
+/** Parse a certificate encoded with <b>len</b> bytes in <b>encoded</b>. */
+tor_cert_t *
+tor_cert_parse(const uint8_t *encoded, const size_t len)
+{
+  tor_cert_t *cert = NULL;
+  ed25519_cert_t *parsed = NULL;
+  ssize_t got_len = ed25519_cert_parse(&parsed, encoded, len);
+  if (got_len < 0 || (size_t) got_len != len)
+    goto err;
+
+  cert = tor_malloc_zero(sizeof(tor_cert_t));
+  cert->encoded = tor_memdup(encoded, len);
+  cert->encoded_len = len;
+
+  memcpy(cert->signed_key.pubkey, parsed->certified_key, 32);
+  int64_t valid_until_64 = ((int64_t)parsed->exp_field) * 3600;
+#if SIZEOF_TIME_T < 8
+  if (valid_until_64 > TIME_MAX)
+    valid_until_64 = TIME_MAX - 1;
+#endif
+  cert->valid_until = (time_t) valid_until_64;
+  cert->cert_type = parsed->cert_type;
+
+  for (unsigned i = 0; i < ed25519_cert_getlen_ext(parsed); ++i) {
+    ed25519_cert_extension_t *ext = ed25519_cert_get_ext(parsed, i);
+    if (ext->ext_type == CERTEXT_SIGNED_WITH_KEY) {
+      if (cert->signing_key_included)
+        goto err;
+
+      cert->signing_key_included = 1;
+      memcpy(cert->signing_key.pubkey, ext->un_signing_key, 32);
+    } else if (ext->ext_flags & CERTEXT_FLAG_AFFECTS_VALIDATION) {
+      /* Unrecognized extension with affects_validation set */
+      goto err;
+    }
+  }
+
+  goto done;
+ err:
+  tor_cert_free(cert);
+  cert = NULL;
+ done:
+  ed25519_cert_free(parsed);
+  return cert;
+}
+
+/** Fill in <b>checkable_out</b> with the information needed to check
+ * the signature on <b>cert</b> with <b>pubkey</b>.
+ *
+ * On success, if <b>expiration_out</b> is provided, and it is some time
+ * _after_ the expiration time of this certificate, set it to the
+ * expiration time of this certificate.
+ */
+int
+tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
+                           const tor_cert_t *cert,
+                           const ed25519_public_key_t *pubkey,
+                           time_t *expiration_out)
+{
+  if (! pubkey) {
+    if (cert->signing_key_included)
+      pubkey = &cert->signing_key;
+    else
+      return -1;
+  }
+
+  checkable_out->msg = cert->encoded;
+  checkable_out->pubkey = pubkey;
+  tor_assert(cert->encoded_len > ED25519_SIG_LEN);
+  const size_t signed_len = cert->encoded_len - ED25519_SIG_LEN;
+  checkable_out->len = signed_len;
+  memcpy(checkable_out->signature.sig,
+         cert->encoded + signed_len, ED25519_SIG_LEN);
+
+  if (expiration_out) {
+    *expiration_out = MIN(*expiration_out, cert->valid_until);
+  }
+
+  return 0;
+}
+
+/** Validates the signature on <b>cert</b> with <b>pubkey</b> relative to the
+ * current time <b>now</b>.  (If <b>now</b> is 0, do not check the expiration
+ * time.) Return 0 on success, -1 on failure.  Sets flags in <b>cert</b> as
+ * appropriate.
+ */
+int
+tor_cert_checksig(tor_cert_t *cert,
+                  const ed25519_public_key_t *pubkey, time_t now)
+{
+  ed25519_checkable_t checkable;
+  int okay;
+  time_t expires = TIME_MAX;
+
+  if (tor_cert_get_checkable_sig(&checkable, cert, pubkey, &expires) < 0)
+    return -1;
+
+  if (now && now > expires) {
+    cert->cert_expired = 1;
+    return -1;
+  }
+
+  if (ed25519_checksig_batch(&okay, &checkable, 1) < 0) {
+    cert->sig_bad = 1;
+    return -1;
+  } else {
+    cert->sig_ok = 1;
+    /* Only copy the checkable public key when it is different from the signing
+     * key of the certificate to avoid undefined behavior. */
+    if (cert->signing_key.pubkey != checkable.pubkey->pubkey) {
+      memcpy(cert->signing_key.pubkey, checkable.pubkey->pubkey, 32);
+    }
+    cert->cert_valid = 1;
+    return 0;
+  }
+}
+
+/** Return a string describing the status of the signature on <b>cert</b>
+ *
+ * Will always be "unchecked" unless tor_cert_checksig has been called.
+ */
+const char *
+tor_cert_describe_signature_status(const tor_cert_t *cert)
+{
+  if (cert->cert_expired) {
+    return "expired";
+  } else if (cert->sig_bad) {
+    return "mis-signed";
+  } else if (cert->sig_ok) {
+    return "okay";
+  } else {
+    return "unchecked";
+  }
+}
+
+/** Return a new copy of <b>cert</b> */
+tor_cert_t *
+tor_cert_dup(const tor_cert_t *cert)
+{
+  tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t));
+  if (cert->encoded)
+    newcert->encoded = tor_memdup(cert->encoded, cert->encoded_len);
+  return newcert;
+}
+
+/** Return true iff cert1 and cert2 are the same cert. */
+int
+tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
+{
+  tor_assert(cert1);
+  tor_assert(cert2);
+  return cert1->encoded_len == cert2->encoded_len &&
+    tor_memeq(cert1->encoded, cert2->encoded, cert1->encoded_len);
+}
+
+/** Return true iff cert1 and cert2 are the same cert, or if they are both
+ * NULL. */
+int
+tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
+{
+  if (cert1 == NULL && cert2 == NULL)
+    return 1;
+  if (!cert1 || !cert2)
+    return 0;
+  return tor_cert_eq(cert1, cert2);
+}
+
+#define RSA_ED_CROSSCERT_PREFIX "Tor TLS RSA/Ed25519 cross-certificate"
+
+/** Create new cross-certification object to certify <b>ed_key</b> as the
+ * master ed25519 identity key for the RSA identity key <b>rsa_key</b>.
+ * Allocates and stores the encoded certificate in *<b>cert</b>, and returns
+ * the number of bytes stored. Returns negative on error.*/
+ssize_t
+tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
+                               const crypto_pk_t *rsa_key,
+                               time_t expires,
+                               uint8_t **cert)
+{
+  // It is later than 1985, since otherwise there would be no C89
+  // compilers. (Try to diagnose #22466.)
+  tor_assert_nonfatal(expires >= 15 * 365 * 86400);
+
+  uint8_t *res;
+
+  rsa_ed_crosscert_t *cc = rsa_ed_crosscert_new();
+  memcpy(cc->ed_key, ed_key->pubkey, ED25519_PUBKEY_LEN);
+  cc->expiration = (uint32_t) CEIL_DIV(expires, 3600);
+  cc->sig_len = crypto_pk_keysize(rsa_key);
+  rsa_ed_crosscert_setlen_sig(cc, crypto_pk_keysize(rsa_key));
+
+  ssize_t alloc_sz = rsa_ed_crosscert_encoded_len(cc);
+  tor_assert(alloc_sz > 0);
+  res = tor_malloc_zero(alloc_sz);
+  ssize_t sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
+  tor_assert(sz > 0 && sz <= alloc_sz);
+
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+
+  const int signed_part_len = 32 + 4;
+  crypto_digest_add_bytes(d, (char*)res, signed_part_len);
+
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
+  int siglen = crypto_pk_private_sign(rsa_key,
+                                      (char*)rsa_ed_crosscert_getarray_sig(cc),
+                                      rsa_ed_crosscert_getlen_sig(cc),
+                                      (char*)digest, sizeof(digest));
+  tor_assert(siglen > 0 && siglen <= (int)crypto_pk_keysize(rsa_key));
+  tor_assert(siglen <= UINT8_MAX);
+  cc->sig_len = siglen;
+  rsa_ed_crosscert_setlen_sig(cc, siglen);
+
+  sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
+  rsa_ed_crosscert_free(cc);
+  *cert = res;
+  return sz;
+}
+
+/**
+ * Check whether the <b>crosscert_len</b> byte certificate in <b>crosscert</b>
+ * is in fact a correct cross-certification of <b>master_key</b> using
+ * the RSA key <b>rsa_id_key</b>.
+ *
+ * Also reject the certificate if it expired before
+ * <b>reject_if_expired_before</b>.
+ *
+ * Return 0 on success, negative on failure.
+ */
+MOCK_IMPL(int,
+rsa_ed25519_crosscert_check, (const uint8_t *crosscert,
+                              const size_t crosscert_len,
+                              const crypto_pk_t *rsa_id_key,
+                              const ed25519_public_key_t *master_key,
+                              const time_t reject_if_expired_before))
+{
+  rsa_ed_crosscert_t *cc = NULL;
+  int rv;
+
+#define ERR(code, s)                                            \
+  do {                                                          \
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,                      \
+           "Received a bad RSA->Ed25519 crosscert: %s",         \
+           (s));                                                \
+    rv = (code);                                                \
+    goto err;                                                   \
+  } while (0)
+
+  if (BUG(crypto_pk_keysize(rsa_id_key) > PK_BYTES))
+    return -1;
+
+  if (BUG(!crosscert))
+    return -1;
+
+  ssize_t parsed_len = rsa_ed_crosscert_parse(&cc, crosscert, crosscert_len);
+  if (parsed_len < 0 || crosscert_len != (size_t)parsed_len) {
+    ERR(-2, "Unparseable or overlong crosscert");
+  }
+
+  if (tor_memneq(rsa_ed_crosscert_getarray_ed_key(cc),
+                 master_key->pubkey,
+                 ED25519_PUBKEY_LEN)) {
+    ERR(-3, "Crosscert did not match Ed25519 key");
+  }
+
+  const uint32_t expiration_date = rsa_ed_crosscert_get_expiration(cc);
+  const uint64_t expiration_time = ((uint64_t)expiration_date) * 3600;
+
+  if (reject_if_expired_before < 0 ||
+      expiration_time < (uint64_t)reject_if_expired_before) {
+    ERR(-4, "Crosscert is expired");
+  }
+
+  const uint8_t *eos = rsa_ed_crosscert_get_end_of_signed(cc);
+  const uint8_t *sig = rsa_ed_crosscert_getarray_sig(cc);
+  const uint8_t siglen = rsa_ed_crosscert_get_sig_len(cc);
+  tor_assert(eos >= crosscert);
+  tor_assert((size_t)(eos - crosscert) <= crosscert_len);
+  tor_assert(siglen == rsa_ed_crosscert_getlen_sig(cc));
+
+  /* Compute the digest */
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+  crypto_digest_add_bytes(d, (char*)crosscert, eos-crosscert);
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
+  /* Now check the signature */
+  uint8_t signed_[PK_BYTES];
+  int signed_len = crypto_pk_public_checksig(rsa_id_key,
+                                          (char*)signed_, sizeof(signed_),
+                                          (char*)sig, siglen);
+  if (signed_len < DIGEST256_LEN) {
+    ERR(-5, "Bad signature, or length of signed data not as expected");
+  }
+
+  if (tor_memneq(digest, signed_, DIGEST256_LEN)) {
+    ERR(-6, "The signature was good, but it didn't match the data");
+  }
+
+  rv = 0;
+ err:
+  rsa_ed_crosscert_free(cc);
+  return rv;
+}
+
+/** Construct and return a new empty or_handshake_certs object */
+or_handshake_certs_t *
+or_handshake_certs_new(void)
+{
+  return tor_malloc_zero(sizeof(or_handshake_certs_t));
+}
+
+/** Release all storage held in <b>certs</b> */
+void
+or_handshake_certs_free_(or_handshake_certs_t *certs)
+{
+  if (!certs)
+    return;
+
+  tor_x509_cert_free(certs->auth_cert);
+  tor_x509_cert_free(certs->link_cert);
+  tor_x509_cert_free(certs->id_cert);
+
+  tor_cert_free(certs->ed_id_sign);
+  tor_cert_free(certs->ed_sign_link);
+  tor_cert_free(certs->ed_sign_auth);
+  tor_free(certs->ed_rsa_crosscert);
+
+  memwipe(certs, 0xBD, sizeof(*certs));
+  tor_free(certs);
+}
+
+#undef ERR
+#define ERR(s)                                                  \
+  do {                                                          \
+    log_fn(severity, LD_PROTOCOL,                               \
+           "Received a bad CERTS cell: %s",                     \
+           (s));                                                \
+    return 0;                                                   \
+  } while (0)
+
+int
+or_handshake_certs_rsa_ok(int severity,
+                          or_handshake_certs_t *certs,
+                          tor_tls_t *tls,
+                          time_t now)
+{
+  tor_x509_cert_t *link_cert = certs->link_cert;
+  tor_x509_cert_t *auth_cert = certs->auth_cert;
+  tor_x509_cert_t *id_cert = certs->id_cert;
+
+  if (certs->started_here) {
+    if (! (id_cert && link_cert))
+      ERR("The certs we wanted (ID, Link) were missing");
+    if (! tor_tls_cert_matches_key(tls, link_cert))
+      ERR("The link certificate didn't match the TLS public key");
+    if (! tor_tls_cert_is_valid(severity, link_cert, id_cert, now, 0))
+      ERR("The link certificate was not valid");
+    if (! tor_tls_cert_is_valid(severity, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  } else {
+    if (! (id_cert && auth_cert))
+      ERR("The certs we wanted (ID, Auth) were missing");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, auth_cert, id_cert, now, 1))
+      ERR("The authentication certificate was not valid");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  }
+
+  return 1;
+}
+
+/** Check all the ed25519 certificates in <b>certs</b> against each other, and
+ * against the peer certificate in <b>tls</b> if appropriate.  On success,
+ * return 0; on failure, return a negative value and warn at level
+ * <b>severity</b> */
+int
+or_handshake_certs_ed25519_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now)
+{
+  ed25519_checkable_t check[10];
+  unsigned n_checkable = 0;
+  time_t expiration = TIME_MAX;
+
+#define ADDCERT(cert, pk)                                               \
+  do {                                                                  \
+    tor_assert(n_checkable < ARRAY_LENGTH(check));                      \
+    if (tor_cert_get_checkable_sig(&check[n_checkable++], cert, pk,     \
+                                   &expiration) < 0)                    \
+      ERR("Could not get checkable cert.");                             \
+  } while (0)
+
+  if (! certs->ed_id_sign || !certs->ed_id_sign->signing_key_included) {
+    ERR("No Ed25519 signing key");
+  }
+  ADDCERT(certs->ed_id_sign, NULL);
+
+  if (certs->started_here) {
+    if (! certs->ed_sign_link)
+      ERR("No Ed25519 link key");
+    {
+      /* check for a match with the TLS cert. */
+      tor_x509_cert_t *peer_cert = tor_tls_get_peer_cert(tls);
+      if (BUG(!peer_cert)) {
+        /* This is a bug, because if we got to this point, we are a connection
+         * that was initiated here, and we completed a TLS handshake. The
+         * other side *must* have given us a certificate! */
+        ERR("No x509 peer cert"); // LCOV_EXCL_LINE
+      }
+      const common_digests_t *peer_cert_digests =
+        tor_x509_cert_get_cert_digests(peer_cert);
+      int okay = tor_memeq(peer_cert_digests->d[DIGEST_SHA256],
+                           certs->ed_sign_link->signed_key.pubkey,
+                           DIGEST256_LEN);
+      tor_x509_cert_free(peer_cert);
+      if (!okay)
+        ERR("Link certificate does not match TLS certificate");
+    }
+
+    ADDCERT(certs->ed_sign_link, &certs->ed_id_sign->signed_key);
+
+  } else {
+    if (! certs->ed_sign_auth)
+      ERR("No Ed25519 link authentication key");
+    ADDCERT(certs->ed_sign_auth, &certs->ed_id_sign->signed_key);
+  }
+
+  if (expiration < now) {
+    ERR("At least one certificate expired.");
+  }
+
+  /* Okay, we've gotten ready to check all the Ed25519 certificates.
+   * Now, we are going to check the RSA certificate's cross-certification
+   * with the ED certificates.
+   *
+   * FFFF In the future, we might want to make this optional.
+   */
+
+  tor_x509_cert_t *rsa_id_cert = certs->id_cert;
+  if (!rsa_id_cert) {
+    ERR("Missing legacy RSA ID certificate");
+  }
+  if (! tor_tls_cert_is_valid(severity, rsa_id_cert, rsa_id_cert, now, 1)) {
+    ERR("The legacy RSA ID certificate was not valid");
+  }
+  if (! certs->ed_rsa_crosscert) {
+    ERR("Missing RSA->Ed25519 crosscert");
+  }
+  crypto_pk_t *rsa_id_key = tor_tls_cert_get_key(rsa_id_cert);
+  if (!rsa_id_key) {
+    ERR("RSA ID cert had no RSA key");
+  }
+
+  if (rsa_ed25519_crosscert_check(certs->ed_rsa_crosscert,
+                                  certs->ed_rsa_crosscert_len,
+                                  rsa_id_key,
+                                  &certs->ed_id_sign->signing_key,
+                                  now) < 0) {
+    crypto_pk_free(rsa_id_key);
+    ERR("Invalid RSA->Ed25519 crosscert");
+  }
+  crypto_pk_free(rsa_id_key);
+  rsa_id_key = NULL;
+
+  /* FFFF We could save a little time in the client case by queueing
+   * this batch to check it later, along with the signature from the
+   * AUTHENTICATE cell. That will change our data flow a bit, though,
+   * so I say "postpone". */
+
+  if (ed25519_checksig_batch(NULL, check, n_checkable) < 0) {
+    ERR("At least one Ed25519 certificate was badly signed");
+  }
+
+  return 1;
+}
+
+/**
+ * Check the Ed certificates and/or the RSA certificates, as appropriate.  If
+ * we obtained an Ed25519 identity, set *ed_id_out. If we obtained an RSA
+ * identity, set *rs_id_out. Otherwise, set them both to NULL.
+ */
+void
+or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out)
+{
+  tor_assert(ed_id_out);
+  tor_assert(rsa_id_out);
+
+  *ed_id_out = NULL;
+  *rsa_id_out = NULL;
+
+  if (certs->ed_id_sign) {
+    if (or_handshake_certs_ed25519_ok(severity, certs, tls, now)) {
+      tor_assert(certs->ed_id_sign);
+      tor_assert(certs->id_cert);
+
+      *ed_id_out = &certs->ed_id_sign->signing_key;
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+
+      /* If we reached this point, we did not look at any of the
+       * subsidiary RSA certificates, so we'd better just remove them.
+       */
+      tor_x509_cert_free(certs->link_cert);
+      tor_x509_cert_free(certs->auth_cert);
+      certs->link_cert = certs->auth_cert = NULL;
+    }
+    /* We do _not_ fall through here.  If you provided us Ed25519
+     * certificates, we expect to verify them! */
+  } else {
+    /* No ed25519 keys given in the CERTS cell */
+    if (or_handshake_certs_rsa_ok(severity, certs, tls, now)) {
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+    }
+  }
+}
+
+/* === ENCODING === */
+
+/* Encode the ed25519 certificate <b>cert</b> and put the newly allocated
+ * string in <b>cert_str_out</b>. Return 0 on success else a negative value. */
+int
+tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out)
+{
+  int ret = -1;
+  char *ed_cert_b64 = NULL;
+  size_t ed_cert_b64_len;
+
+  tor_assert(cert);
+  tor_assert(cert_str_out);
+
+  /* Get the encoded size and add the NUL byte. */
+  ed_cert_b64_len = base64_encode_size(cert->encoded_len,
+                                       BASE64_ENCODE_MULTILINE) + 1;
+  ed_cert_b64 = tor_malloc_zero(ed_cert_b64_len);
+
+  /* Base64 encode the encoded certificate. */
+  if (base64_encode(ed_cert_b64, ed_cert_b64_len,
+                    (const char *) cert->encoded, cert->encoded_len,
+                    BASE64_ENCODE_MULTILINE) < 0) {
+    /* LCOV_EXCL_START */
+    log_err(LD_BUG, "Couldn't base64-encode ed22519 cert!");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* Put everything together in a NUL terminated string. */
+  tor_asprintf(cert_str_out,
+               "-----BEGIN ED25519 CERT-----\n"
+               "%s"
+               "-----END ED25519 CERT-----",
+               ed_cert_b64);
+  /* Success! */
+  ret = 0;
+
+ err:
+  tor_free(ed_cert_b64);
+  return ret;
+}
diff --git a/src/feature/nodelist/torcert.h b/src/feature/nodelist/torcert.h
new file mode 100644
index 0000000000..5fa97679df
--- /dev/null
+++ b/src/feature/nodelist/torcert.h
@@ -0,0 +1,110 @@
+/* Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TORCERT_H_INCLUDED
+#define TORCERT_H_INCLUDED
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#define SIGNED_KEY_TYPE_ED25519     0x01
+
+#define CERT_TYPE_ID_SIGNING        0x04
+#define CERT_TYPE_SIGNING_LINK      0x05
+#define CERT_TYPE_SIGNING_AUTH      0x06
+#define CERT_TYPE_SIGNING_HS_DESC   0x08
+#define CERT_TYPE_AUTH_HS_IP_KEY    0x09
+#define CERT_TYPE_ONION_ID          0x0A
+#define CERT_TYPE_CROSS_HS_IP_KEYS  0x0B
+
+#define CERT_FLAG_INCLUDE_SIGNING_KEY 0x1
+
+/** An ed25519-signed certificate as used throughout the Tor protocol.
+ **/
+typedef struct tor_cert_st {
+  /** The key authenticated by this certificate */
+  ed25519_public_key_t signed_key;
+  /** The key that signed this certificate. This value may be unset if the
+   * certificate has never been checked, and didn't include its own key. */
+  ed25519_public_key_t signing_key;
+  /** A time after which this certificate will no longer be valid. */
+  time_t valid_until;
+
+  /** The encoded representation of this certificate */
+  uint8_t *encoded;
+  /** The length of <b>encoded</b> */
+  size_t encoded_len;
+
+  /** One of CERT_TYPE_... */
+  uint8_t cert_type;
+  /** True iff we received a signing key embedded in this certificate */
+  unsigned signing_key_included : 1;
+  /** True iff we checked the signature and found it bad */
+  unsigned sig_bad : 1;
+  /** True iff we checked the signature and found it correct */
+  unsigned sig_ok : 1;
+  /** True iff we checked the signature and first found that the cert
+   * had expired */
+  unsigned cert_expired : 1;
+  /** True iff we checked the signature and found the whole cert valid */
+  unsigned cert_valid : 1;
+} tor_cert_t;
+
+struct tor_tls_t;
+
+tor_cert_t *tor_cert_create(const ed25519_keypair_t *signing_key,
+                            uint8_t cert_type,
+                            const ed25519_public_key_t *signed_key,
+                            time_t now, time_t lifetime,
+                            uint32_t flags);
+
+tor_cert_t *tor_cert_parse(const uint8_t *cert, size_t certlen);
+
+void tor_cert_free_(tor_cert_t *cert);
+#define tor_cert_free(cert) FREE_AND_NULL(tor_cert_t, tor_cert_free_, (cert))
+
+int tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
+                               const tor_cert_t *out,
+                               const ed25519_public_key_t *pubkey,
+                               time_t *expiration_out);
+
+int tor_cert_checksig(tor_cert_t *cert,
+                      const ed25519_public_key_t *pubkey, time_t now);
+const char *tor_cert_describe_signature_status(const tor_cert_t *cert);
+
+tor_cert_t *tor_cert_dup(const tor_cert_t *cert);
+int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
+int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
+
+ssize_t tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
+                                       const crypto_pk_t *rsa_key,
+                                       time_t expires,
+                                       uint8_t **cert);
+MOCK_DECL(int,
+rsa_ed25519_crosscert_check, (const uint8_t *crosscert,
+                              const size_t crosscert_len,
+                              const crypto_pk_t *rsa_id_key,
+                              const ed25519_public_key_t *master_key,
+                              const time_t reject_if_expired_before));
+
+or_handshake_certs_t *or_handshake_certs_new(void);
+void or_handshake_certs_free_(or_handshake_certs_t *certs);
+#define or_handshake_certs_free(certs) \
+  FREE_AND_NULL(or_handshake_certs_t, or_handshake_certs_free_, (certs))
+int or_handshake_certs_rsa_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              struct tor_tls_t *tls,
+                              time_t now);
+int or_handshake_certs_ed25519_ok(int severity,
+                                  or_handshake_certs_t *certs,
+                                  struct tor_tls_t *tls,
+                                  time_t now);
+void or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              struct tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out);
+
+int tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out);
+
+#endif /* !defined(TORCERT_H_INCLUDED) */
diff --git a/src/feature/nodelist/vote_routerstatus_st.h b/src/feature/nodelist/vote_routerstatus_st.h
new file mode 100644
index 0000000000..1b85737df8
--- /dev/null
+++ b/src/feature/nodelist/vote_routerstatus_st.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef VOTE_ROUTERSTATUS_ST_H
+#define VOTE_ROUTERSTATUS_ST_H
+
+#include "or/routerstatus_st.h"
+#include "lib/defs/x25519_sizes.h"
+
+/** The claim about a single router, made in a vote. */
+struct vote_routerstatus_t {
+  routerstatus_t status; /**< Underlying 'status' object for this router.
+                          * Flags are redundant. */
+  /** How many known-flags are allowed in a vote? This is the width of
+   * the flags field of vote_routerstatus_t */
+#define MAX_KNOWN_FLAGS_IN_VOTE 64
+  uint64_t flags; /**< Bit-field for all recognized flags; index into
+                   * networkstatus_t.known_flags. */
+  char *version; /**< The version that the authority says this router is
+                  * running. */
+  char *protocols; /**< The protocols that this authority says this router
+                    * provides. */
+  unsigned int has_measured_bw:1; /**< The vote had a measured bw */
+  /** True iff the vote included an entry for ed25519 ID, or included
+   * "id ed25519 none" to indicate that there was no ed25519 ID. */
+  unsigned int has_ed25519_listing:1;
+  /** True if the Ed25519 listing here is the consensus-opinion for the
+   * Ed25519 listing; false if there was no consensus on Ed25519 key status,
+   * or if this VRS doesn't reflect it. */
+  unsigned int ed25519_reflects_consensus:1;
+  uint32_t measured_bw_kb; /**< Measured bandwidth (capacity) of the router */
+  /** The hash or hashes that the authority claims this microdesc has. */
+  vote_microdesc_hash_t *microdesc;
+  /** Ed25519 identity for this router, or zero if it has none. */
+  uint8_t ed25519_id[ED25519_PUBKEY_LEN];
+};
+
+#endif
diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c
new file mode 100644
index 0000000000..4ac58552f4
--- /dev/null
+++ b/src/feature/relay/dns.c
@@ -0,0 +1,2143 @@
+/* Copyright (c) 2003-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dns.c
+ * \brief Implements a local cache for DNS results for Tor servers.
+ * This is implemented as a wrapper around Adam Langley's eventdns.c code.
+ * (We can't just use gethostbyname() and friends because we really need to
+ * be nonblocking.)
+ *
+ * There are three main cases when a Tor relay uses dns.c to launch a DNS
+ * request:
+ *   <ol>
+ *    <li>To check whether the DNS server is working more or less correctly.
+ *      This happens via dns_launch_correctness_checks().  The answer is
+ *      reported in the return value from later calls to
+ *      dns_seems_to_be_broken().
+ *    <li>When a client has asked the relay, in a RELAY_BEGIN cell, to connect
+ *      to a given server by hostname.  This happens via dns_resolve().
+ *    <li>When a client has asked the relay, in a RELAY_RESOLVE cell, to look
+ *      up a given server's IP address(es) by hostname. This also happens via
+ *      dns_resolve().
+ *   </ol>
+ *
+ * Each of these gets handled a little differently.
+ *
+ * To check for correctness, we look up some hostname we expect to exist and
+ * have real entries, some hostnames which we expect to definitely not exist,
+ * and some hostnames that we expect to probably not exist.  If too many of
+ * the hostnames that shouldn't exist do exist, that's a DNS hijacking
+ * attempt.  If too many of the hostnames that should exist have the same
+ * addresses as the ones that shouldn't exist, that's a very bad DNS hijacking
+ * attempt, or a very naughty captive portal.  And if the hostnames that
+ * should exist simply don't exist, we probably have a broken nameserver.
+ *
+ * To handle client requests, we first check our cache for answers. If there
+ * isn't something up-to-date, we've got to launch A or AAAA requests as
+ * appropriate.  How we handle responses to those in particular is a bit
+ * complex; see dns_lookup() and set_exitconn_info_from_resolve().
+ *
+ * When a lookup is finally complete, the inform_pending_connections()
+ * function will tell all of the streams that have been waiting for the
+ * resolve, by calling connection_exit_connect() if the client sent a
+ * RELAY_BEGIN cell, and by calling send_resolved_cell() or
+ * send_hostname_cell() if the client sent a RELAY_RESOLVE cell.
+ **/
+
+#define DNS_PRIVATE
+
+#include "or/or.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/connection_edge.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "or/dns.h"
+#include "or/main.h"
+#include "or/policies.h"
+#include "or/relay.h"
+#include "or/router.h"
+#include "ht.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include "or/edge_connection_st.h"
+#include "or/or_circuit_st.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#include <event2/event.h>
+#include <event2/dns.h>
+
+/** How long will we wait for an answer from the resolver before we decide
+ * that the resolver is wedged? */
+#define RESOLVE_MAX_TIMEOUT 300
+
+/** Our evdns_base; this structure handles all our name lookups. */
+static struct evdns_base *the_evdns_base = NULL;
+
+/** Have we currently configured nameservers with eventdns? */
+static int nameservers_configured = 0;
+/** Did our most recent attempt to configure nameservers with eventdns fail? */
+static int nameserver_config_failed = 0;
+/** What was the resolv_conf fname we last used when configuring the
+ * nameservers? Used to check whether we need to reconfigure. */
+static char *resolv_conf_fname = NULL;
+/** What was the mtime on the resolv.conf file we last used when configuring
+ * the nameservers?  Used to check whether we need to reconfigure. */
+static time_t resolv_conf_mtime = 0;
+
+static void purge_expired_resolves(time_t now);
+static void dns_found_answer(const char *address, uint8_t query_type,
+                             int dns_answer,
+                             const tor_addr_t *addr,
+                             const char *hostname,
+                             uint32_t ttl);
+static void add_wildcarded_test_address(const char *address);
+static int configure_nameservers(int force);
+static int answer_is_wildcarded(const char *ip);
+static int evdns_err_is_transient(int err);
+static void inform_pending_connections(cached_resolve_t *resolve);
+static void make_pending_resolve_cached(cached_resolve_t *cached);
+
+#ifdef DEBUG_DNS_CACHE
+static void assert_cache_ok_(void);
+#define assert_cache_ok() assert_cache_ok_()
+#else
+#define assert_cache_ok() STMT_NIL
+#endif /* defined(DEBUG_DNS_CACHE) */
+static void assert_resolve_ok(cached_resolve_t *resolve);
+
+/** Hash table of cached_resolve objects. */
+static HT_HEAD(cache_map, cached_resolve_t) cache_root;
+
+/** Global: how many IPv6 requests have we made in all? */
+static uint64_t n_ipv6_requests_made = 0;
+/** Global: how many IPv6 requests have timed out? */
+static uint64_t n_ipv6_timeouts = 0;
+/** Global: Do we think that IPv6 DNS is broken? */
+static int dns_is_broken_for_ipv6 = 0;
+
+/** Function to compare hashed resolves on their addresses; used to
+ * implement hash tables. */
+static inline int
+cached_resolves_eq(cached_resolve_t *a, cached_resolve_t *b)
+{
+  /* make this smarter one day? */
+  assert_resolve_ok(a); // Not b; b may be just a search.
+  return !strncmp(a->address, b->address, MAX_ADDRESSLEN);
+}
+
+/** Hash function for cached_resolve objects */
+static inline unsigned int
+cached_resolve_hash(cached_resolve_t *a)
+{
+  return (unsigned) siphash24g((const uint8_t*)a->address, strlen(a->address));
+}
+
+HT_PROTOTYPE(cache_map, cached_resolve_t, node, cached_resolve_hash,
+             cached_resolves_eq)
+HT_GENERATE2(cache_map, cached_resolve_t, node, cached_resolve_hash,
+             cached_resolves_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/** Initialize the DNS cache. */
+static void
+init_cache_map(void)
+{
+  HT_INIT(cache_map, &cache_root);
+}
+
+/** Helper: called by eventdns when eventdns wants to log something. */
+static void
+evdns_log_cb(int warn, const char *msg)
+{
+  const char *cp;
+  static int all_down = 0;
+  int severity = warn ? LOG_WARN : LOG_INFO;
+  if (!strcmpstart(msg, "Resolve requested for") &&
+      get_options()->SafeLogging) {
+    log_info(LD_EXIT, "eventdns: Resolve requested.");
+    return;
+  } else if (!strcmpstart(msg, "Search: ")) {
+    return;
+  }
+  if (!strcmpstart(msg, "Nameserver ") && (cp=strstr(msg, " has failed: "))) {
+    char *ns = tor_strndup(msg+11, cp-(msg+11));
+    const char *colon = strchr(cp, ':');
+    tor_assert(colon);
+    const char *err = colon+2;
+    /* Don't warn about a single failed nameserver; we'll warn with 'all
+     * nameservers have failed' if we're completely out of nameservers;
+     * otherwise, the situation is tolerable. */
+    severity = LOG_INFO;
+    control_event_server_status(LOG_NOTICE,
+                                "NAMESERVER_STATUS NS=%s STATUS=DOWN ERR=%s",
+                                ns, escaped(err));
+    tor_free(ns);
+  } else if (!strcmpstart(msg, "Nameserver ") &&
+             (cp=strstr(msg, " is back up"))) {
+    char *ns = tor_strndup(msg+11, cp-(msg+11));
+    severity = (all_down && warn) ? LOG_NOTICE : LOG_INFO;
+    all_down = 0;
+    control_event_server_status(LOG_NOTICE,
+                                "NAMESERVER_STATUS NS=%s STATUS=UP", ns);
+    tor_free(ns);
+  } else if (!strcmp(msg, "All nameservers have failed")) {
+    control_event_server_status(LOG_WARN, "NAMESERVER_ALL_DOWN");
+    all_down = 1;
+  } else if (!strcmpstart(msg, "Address mismatch on received DNS")) {
+    static ratelim_t mismatch_limit = RATELIM_INIT(3600);
+    const char *src = strstr(msg, " Apparent source");
+    if (!src || get_options()->SafeLogging) {
+      src = "";
+    }
+    log_fn_ratelim(&mismatch_limit, severity, LD_EXIT,
+                   "eventdns: Received a DNS packet from "
+                   "an IP address to which we did not send a request. This "
+                   "could be a DNS spoofing attempt, or some kind of "
+                   "misconfiguration.%s", src);
+    return;
+  }
+  tor_log(severity, LD_EXIT, "eventdns: %s", msg);
+}
+
+/** Helper: passed to eventdns.c as a callback so it can generate random
+ * numbers for transaction IDs and 0x20-hack coding. */
+static void
+dns_randfn_(char *b, size_t n)
+{
+  crypto_rand(b,n);
+}
+
+/** Initialize the DNS subsystem; called by the OR process. */
+int
+dns_init(void)
+{
+  init_cache_map();
+  evdns_set_random_bytes_fn(dns_randfn_);
+  if (server_mode(get_options())) {
+    int r = configure_nameservers(1);
+    return r;
+  }
+  return 0;
+}
+
+/** Called when DNS-related options change (or may have changed).  Returns -1
+ * on failure, 0 on success. */
+int
+dns_reset(void)
+{
+  const or_options_t *options = get_options();
+  if (! server_mode(options)) {
+
+    if (!the_evdns_base) {
+      if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) {
+        log_err(LD_BUG, "Couldn't create an evdns_base");
+        return -1;
+      }
+    }
+
+    evdns_base_clear_nameservers_and_suspend(the_evdns_base);
+    evdns_base_search_clear(the_evdns_base);
+    nameservers_configured = 0;
+    tor_free(resolv_conf_fname);
+    resolv_conf_mtime = 0;
+  } else {
+    if (configure_nameservers(0) < 0) {
+      return -1;
+    }
+  }
+  return 0;
+}
+
+/** Return true iff the most recent attempt to initialize the DNS subsystem
+ * failed. */
+int
+has_dns_init_failed(void)
+{
+  return nameserver_config_failed;
+}
+
+/** Helper: Given a TTL from a DNS response, determine what TTL to give the
+ * OP that asked us to resolve it, and how long to cache that record
+ * ourselves. */
+uint32_t
+dns_clip_ttl(uint32_t ttl)
+{
+  /* This logic is a defense against "DefectTor" DNS-based traffic
+   * confirmation attacks, as in https://nymity.ch/tor-dns/tor-dns.pdf .
+   * We only give two values: a "low" value and a "high" value.
+   */
+  if (ttl < MIN_DNS_TTL_AT_EXIT)
+    return MIN_DNS_TTL_AT_EXIT;
+  else
+    return MAX_DNS_TTL_AT_EXIT;
+}
+
+/** Helper: free storage held by an entry in the DNS cache. */
+static void
+free_cached_resolve_(cached_resolve_t *r)
+{
+  if (!r)
+    return;
+  while (r->pending_connections) {
+    pending_connection_t *victim = r->pending_connections;
+    r->pending_connections = victim->next;
+    tor_free(victim);
+  }
+  if (r->res_status_hostname == RES_STATUS_DONE_OK)
+    tor_free(r->result_ptr.hostname);
+  r->magic = 0xFF00FF00;
+  tor_free(r);
+}
+
+/** Compare two cached_resolve_t pointers by expiry time, and return
+ * less-than-zero, zero, or greater-than-zero as appropriate. Used for
+ * the priority queue implementation. */
+static int
+compare_cached_resolves_by_expiry_(const void *_a, const void *_b)
+{
+  const cached_resolve_t *a = _a, *b = _b;
+  if (a->expire < b->expire)
+    return -1;
+  else if (a->expire == b->expire)
+    return 0;
+  else
+    return 1;
+}
+
+/** Priority queue of cached_resolve_t objects to let us know when they
+ * will expire. */
+static smartlist_t *cached_resolve_pqueue = NULL;
+
+static void
+cached_resolve_add_answer(cached_resolve_t *resolve,
+                          int query_type,
+                          int dns_result,
+                          const tor_addr_t *answer_addr,
+                          const char *answer_hostname,
+                          uint32_t ttl)
+{
+  if (query_type == DNS_PTR) {
+    if (resolve->res_status_hostname != RES_STATUS_INFLIGHT)
+      return;
+
+    if (dns_result == DNS_ERR_NONE && answer_hostname) {
+      resolve->result_ptr.hostname = tor_strdup(answer_hostname);
+      resolve->res_status_hostname = RES_STATUS_DONE_OK;
+    } else {
+      resolve->result_ptr.err_hostname = dns_result;
+      resolve->res_status_hostname = RES_STATUS_DONE_ERR;
+    }
+    resolve->ttl_hostname = ttl;
+  } else if (query_type == DNS_IPv4_A) {
+    if (resolve->res_status_ipv4 != RES_STATUS_INFLIGHT)
+      return;
+
+    if (dns_result == DNS_ERR_NONE && answer_addr &&
+        tor_addr_family(answer_addr) == AF_INET) {
+      resolve->result_ipv4.addr_ipv4 = tor_addr_to_ipv4h(answer_addr);
+      resolve->res_status_ipv4 = RES_STATUS_DONE_OK;
+    } else {
+      resolve->result_ipv4.err_ipv4 = dns_result;
+      resolve->res_status_ipv4 = RES_STATUS_DONE_ERR;
+    }
+    resolve->ttl_ipv4 = ttl;
+  } else if (query_type == DNS_IPv6_AAAA) {
+    if (resolve->res_status_ipv6 != RES_STATUS_INFLIGHT)
+      return;
+
+    if (dns_result == DNS_ERR_NONE && answer_addr &&
+        tor_addr_family(answer_addr) == AF_INET6) {
+      memcpy(&resolve->result_ipv6.addr_ipv6,
+             tor_addr_to_in6(answer_addr),
+             sizeof(struct in6_addr));
+      resolve->res_status_ipv6 = RES_STATUS_DONE_OK;
+    } else {
+      resolve->result_ipv6.err_ipv6 = dns_result;
+      resolve->res_status_ipv6 = RES_STATUS_DONE_ERR;
+    }
+    resolve->ttl_ipv6 = ttl;
+  }
+}
+
+/** Return true iff there are no in-flight requests for <b>resolve</b>. */
+static int
+cached_resolve_have_all_answers(const cached_resolve_t *resolve)
+{
+  return (resolve->res_status_ipv4 != RES_STATUS_INFLIGHT &&
+          resolve->res_status_ipv6 != RES_STATUS_INFLIGHT &&
+          resolve->res_status_hostname != RES_STATUS_INFLIGHT);
+}
+
+/** Set an expiry time for a cached_resolve_t, and add it to the expiry
+ * priority queue */
+static void
+set_expiry(cached_resolve_t *resolve, time_t expires)
+{
+  tor_assert(resolve && resolve->expire == 0);
+  if (!cached_resolve_pqueue)
+    cached_resolve_pqueue = smartlist_new();
+  resolve->expire = expires;
+  smartlist_pqueue_add(cached_resolve_pqueue,
+                       compare_cached_resolves_by_expiry_,
+                       offsetof(cached_resolve_t, minheap_idx),
+                       resolve);
+}
+
+/** Free all storage held in the DNS cache and related structures. */
+void
+dns_free_all(void)
+{
+  cached_resolve_t **ptr, **next, *item;
+  assert_cache_ok();
+  if (cached_resolve_pqueue) {
+    SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res,
+      {
+        if (res->state == CACHE_STATE_DONE)
+          free_cached_resolve_(res);
+      });
+  }
+  for (ptr = HT_START(cache_map, &cache_root); ptr != NULL; ptr = next) {
+    item = *ptr;
+    next = HT_NEXT_RMV(cache_map, &cache_root, ptr);
+    free_cached_resolve_(item);
+  }
+  HT_CLEAR(cache_map, &cache_root);
+  smartlist_free(cached_resolve_pqueue);
+  cached_resolve_pqueue = NULL;
+  tor_free(resolv_conf_fname);
+}
+
+/** Remove every cached_resolve whose <b>expire</b> time is before or
+ * equal to <b>now</b> from the cache. */
+static void
+purge_expired_resolves(time_t now)
+{
+  cached_resolve_t *resolve, *removed;
+  pending_connection_t *pend;
+  edge_connection_t *pendconn;
+
+  assert_cache_ok();
+  if (!cached_resolve_pqueue)
+    return;
+
+  while (smartlist_len(cached_resolve_pqueue)) {
+    resolve = smartlist_get(cached_resolve_pqueue, 0);
+    if (resolve->expire > now)
+      break;
+    smartlist_pqueue_pop(cached_resolve_pqueue,
+                         compare_cached_resolves_by_expiry_,
+                         offsetof(cached_resolve_t, minheap_idx));
+
+    if (resolve->state == CACHE_STATE_PENDING) {
+      log_debug(LD_EXIT,
+                "Expiring a dns resolve %s that's still pending. Forgot to "
+                "cull it? DNS resolve didn't tell us about the timeout?",
+                escaped_safe_str(resolve->address));
+    } else if (resolve->state == CACHE_STATE_CACHED) {
+      log_debug(LD_EXIT,
+                "Forgetting old cached resolve (address %s, expires %lu)",
+                escaped_safe_str(resolve->address),
+                (unsigned long)resolve->expire);
+      tor_assert(!resolve->pending_connections);
+    } else {
+      tor_assert(resolve->state == CACHE_STATE_DONE);
+      tor_assert(!resolve->pending_connections);
+    }
+
+    if (resolve->pending_connections) {
+      log_debug(LD_EXIT,
+                "Closing pending connections on timed-out DNS resolve!");
+      while (resolve->pending_connections) {
+        pend = resolve->pending_connections;
+        resolve->pending_connections = pend->next;
+        /* Connections should only be pending if they have no socket. */
+        tor_assert(!SOCKET_OK(pend->conn->base_.s));
+        pendconn = pend->conn;
+        /* Prevent double-remove */
+        pendconn->base_.state = EXIT_CONN_STATE_RESOLVEFAILED;
+        if (!pendconn->base_.marked_for_close) {
+          connection_edge_end(pendconn, END_STREAM_REASON_TIMEOUT);
+          circuit_detach_stream(circuit_get_by_edge_conn(pendconn), pendconn);
+          connection_free_(TO_CONN(pendconn));
+        }
+        tor_free(pend);
+      }
+    }
+
+    if (resolve->state == CACHE_STATE_CACHED ||
+        resolve->state == CACHE_STATE_PENDING) {
+      removed = HT_REMOVE(cache_map, &cache_root, resolve);
+      if (removed != resolve) {
+        log_err(LD_BUG, "The expired resolve we purged didn't match any in"
+                " the cache. Tried to purge %s (%p); instead got %s (%p).",
+                resolve->address, (void*)resolve,
+                removed ? removed->address : "NULL", (void*)removed);
+      }
+      tor_assert(removed == resolve);
+    } else {
+      /* This should be in state DONE. Make sure it's not in the cache. */
+      cached_resolve_t *tmp = HT_FIND(cache_map, &cache_root, resolve);
+      tor_assert(tmp != resolve);
+    }
+    if (resolve->res_status_hostname == RES_STATUS_DONE_OK)
+      tor_free(resolve->result_ptr.hostname);
+    resolve->magic = 0xF0BBF0BB;
+    tor_free(resolve);
+  }
+
+  assert_cache_ok();
+}
+
+/* argument for send_resolved_cell only, meaning "let the answer type be ipv4
+ * or ipv6 depending on the connection's address". */
+#define RESOLVED_TYPE_AUTO 0xff
+
+/** Send a response to the RESOLVE request of a connection.
+ * <b>answer_type</b> must be one of
+ * RESOLVED_TYPE_(AUTO|ERROR|ERROR_TRANSIENT|).
+ *
+ * If <b>circ</b> is provided, and we have a cached answer, send the
+ * answer back along circ; otherwise, send the answer back along
+ * <b>conn</b>'s attached circuit.
+ */
+MOCK_IMPL(STATIC void,
+send_resolved_cell,(edge_connection_t *conn, uint8_t answer_type,
+                    const cached_resolve_t *resolved))
+{
+  char buf[RELAY_PAYLOAD_SIZE], *cp = buf;
+  size_t buflen = 0;
+  uint32_t ttl;
+
+  buf[0] = answer_type;
+  ttl = dns_clip_ttl(conn->address_ttl);
+
+  switch (answer_type)
+    {
+    case RESOLVED_TYPE_AUTO:
+      if (resolved && resolved->res_status_ipv4 == RES_STATUS_DONE_OK) {
+        cp[0] = RESOLVED_TYPE_IPV4;
+        cp[1] = 4;
+        set_uint32(cp+2, htonl(resolved->result_ipv4.addr_ipv4));
+        set_uint32(cp+6, htonl(ttl));
+        cp += 10;
+      }
+      if (resolved && resolved->res_status_ipv6 == RES_STATUS_DONE_OK) {
+        const uint8_t *bytes = resolved->result_ipv6.addr_ipv6.s6_addr;
+        cp[0] = RESOLVED_TYPE_IPV6;
+        cp[1] = 16;
+        memcpy(cp+2, bytes, 16);
+        set_uint32(cp+18, htonl(ttl));
+        cp += 22;
+      }
+      if (cp != buf) {
+        buflen = cp - buf;
+        break;
+      } else {
+        answer_type = RESOLVED_TYPE_ERROR;
+        /* fall through. */
+      }
+      /* Falls through. */
+    case RESOLVED_TYPE_ERROR_TRANSIENT:
+    case RESOLVED_TYPE_ERROR:
+      {
+        const char *errmsg = "Error resolving hostname";
+        size_t msglen = strlen(errmsg);
+
+        buf[0] = answer_type;
+        buf[1] = msglen;
+        strlcpy(buf+2, errmsg, sizeof(buf)-2);
+        set_uint32(buf+2+msglen, htonl(ttl));
+        buflen = 6+msglen;
+        break;
+      }
+    default:
+      tor_assert(0);
+      return;
+    }
+  // log_notice(LD_EXIT, "Sending a regular RESOLVED reply: ");
+
+  connection_edge_send_command(conn, RELAY_COMMAND_RESOLVED, buf, buflen);
+}
+
+/** Send a response to the RESOLVE request of a connection for an in-addr.arpa
+ * address on connection <b>conn</b> which yielded the result <b>hostname</b>.
+ * The answer type will be RESOLVED_HOSTNAME.
+ *
+ * If <b>circ</b> is provided, and we have a cached answer, send the
+ * answer back along circ; otherwise, send the answer back along
+ * <b>conn</b>'s attached circuit.
+ */
+MOCK_IMPL(STATIC void,
+send_resolved_hostname_cell,(edge_connection_t *conn,
+                             const char *hostname))
+{
+  char buf[RELAY_PAYLOAD_SIZE];
+  size_t buflen;
+  uint32_t ttl;
+  size_t namelen = strlen(hostname);
+  tor_assert(hostname);
+
+  tor_assert(namelen < 256);
+  ttl = dns_clip_ttl(conn->address_ttl);
+
+  buf[0] = RESOLVED_TYPE_HOSTNAME;
+  buf[1] = (uint8_t)namelen;
+  memcpy(buf+2, hostname, namelen);
+  set_uint32(buf+2+namelen, htonl(ttl));
+  buflen = 2+namelen+4;
+
+  // log_notice(LD_EXIT, "Sending a reply RESOLVED reply: %s", hostname);
+  connection_edge_send_command(conn, RELAY_COMMAND_RESOLVED, buf, buflen);
+  // log_notice(LD_EXIT, "Sent");
+}
+
+/** See if we have a cache entry for <b>exitconn</b>-\>address. If so,
+ * if resolve valid, put it into <b>exitconn</b>-\>addr and return 1.
+ * If resolve failed, free exitconn and return -1.
+ *
+ * (For EXIT_PURPOSE_RESOLVE connections, send back a RESOLVED error cell
+ * on returning -1.  For EXIT_PURPOSE_CONNECT connections, there's no
+ * need to send back an END cell, since connection_exit_begin_conn will
+ * do that for us.)
+ *
+ * If we have a cached answer, send the answer back along <b>exitconn</b>'s
+ * circuit.
+ *
+ * Else, if seen before and pending, add conn to the pending list,
+ * and return 0.
+ *
+ * Else, if not seen before, add conn to pending list, hand to
+ * dns farm, and return 0.
+ *
+ * Exitconn's on_circuit field must be set, but exitconn should not
+ * yet be linked onto the n_streams/resolving_streams list of that circuit.
+ * On success, link the connection to n_streams if it's an exit connection.
+ * On "pending", link the connection to resolving streams.  Otherwise,
+ * clear its on_circuit field.
+ */
+int
+dns_resolve(edge_connection_t *exitconn)
+{
+  or_circuit_t *oncirc = TO_OR_CIRCUIT(exitconn->on_circuit);
+  int is_resolve, r;
+  int made_connection_pending = 0;
+  char *hostname = NULL;
+  cached_resolve_t *resolve = NULL;
+  is_resolve = exitconn->base_.purpose == EXIT_PURPOSE_RESOLVE;
+
+  r = dns_resolve_impl(exitconn, is_resolve, oncirc, &hostname,
+                       &made_connection_pending, &resolve);
+
+  switch (r) {
+    case 1:
+      /* We got an answer without a lookup -- either the answer was
+       * cached, or it was obvious (like an IP address). */
+      if (is_resolve) {
+        /* Send the answer back right now, and detach. */
+        if (hostname)
+          send_resolved_hostname_cell(exitconn, hostname);
+        else
+          send_resolved_cell(exitconn, RESOLVED_TYPE_AUTO, resolve);
+        exitconn->on_circuit = NULL;
+      } else {
+        /* Add to the n_streams list; the calling function will send back a
+         * connected cell. */
+        exitconn->next_stream = oncirc->n_streams;
+        oncirc->n_streams = exitconn;
+      }
+      break;
+    case 0:
+      /* The request is pending: add the connection into the linked list of
+       * resolving_streams on this circuit. */
+      exitconn->base_.state = EXIT_CONN_STATE_RESOLVING;
+      exitconn->next_stream = oncirc->resolving_streams;
+      oncirc->resolving_streams = exitconn;
+      break;
+    case -2:
+    case -1:
+      /* The request failed before it could start: cancel this connection,
+       * and stop everybody waiting for the same connection. */
+      if (is_resolve) {
+        send_resolved_cell(exitconn,
+             (r == -1) ? RESOLVED_TYPE_ERROR : RESOLVED_TYPE_ERROR_TRANSIENT,
+             NULL);
+      }
+
+      exitconn->on_circuit = NULL;
+
+      dns_cancel_pending_resolve(exitconn->base_.address);
+
+      if (!made_connection_pending && !exitconn->base_.marked_for_close) {
+        /* If we made the connection pending, then we freed it already in
+         * dns_cancel_pending_resolve().  If we marked it for close, it'll
+         * get freed from the main loop.  Otherwise, can free it now. */
+        connection_free_(TO_CONN(exitconn));
+      }
+      break;
+    default:
+      tor_assert(0);
+  }
+
+  tor_free(hostname);
+  return r;
+}
+
+/** Helper function for dns_resolve: same functionality, but does not handle:
+ *     - marking connections on error and clearing their on_circuit
+ *     - linking connections to n_streams/resolving_streams,
+ *     - sending resolved cells if we have an answer/error right away,
+ *
+ * Return -2 on a transient error. If it's a reverse resolve and it's
+ * successful, sets *<b>hostname_out</b> to a newly allocated string
+ * holding the cached reverse DNS value.
+ *
+ * Set *<b>made_connection_pending_out</b> to true if we have placed
+ * <b>exitconn</b> on the list of pending connections for some resolve; set it
+ * to false otherwise.
+ *
+ * Set *<b>resolve_out</b> to a cached resolve, if we found one.
+ */
+MOCK_IMPL(STATIC int,
+dns_resolve_impl,(edge_connection_t *exitconn, int is_resolve,
+                 or_circuit_t *oncirc, char **hostname_out,
+                 int *made_connection_pending_out,
+                 cached_resolve_t **resolve_out))
+{
+  cached_resolve_t *resolve;
+  cached_resolve_t search;
+  pending_connection_t *pending_connection;
+  int is_reverse = 0;
+  tor_addr_t addr;
+  time_t now = time(NULL);
+  int r;
+  assert_connection_ok(TO_CONN(exitconn), 0);
+  tor_assert(!SOCKET_OK(exitconn->base_.s));
+  assert_cache_ok();
+  tor_assert(oncirc);
+  *made_connection_pending_out = 0;
+
+  /* first check if exitconn->base_.address is an IP. If so, we already
+   * know the answer. */
+  if (tor_addr_parse(&addr, exitconn->base_.address) >= 0) {
+    if (tor_addr_family(&addr) == AF_INET ||
+        tor_addr_family(&addr) == AF_INET6) {
+      tor_addr_copy(&exitconn->base_.addr, &addr);
+      exitconn->address_ttl = DEFAULT_DNS_TTL;
+      return 1;
+    } else {
+      /* XXXX unspec? Bogus? */
+      return -1;
+    }
+  }
+
+  /* If we're a non-exit, don't even do DNS lookups. */
+  if (router_my_exit_policy_is_reject_star())
+    return -1;
+
+  if (address_is_invalid_destination(exitconn->base_.address, 0)) {
+    tor_log(LOG_PROTOCOL_WARN, LD_EXIT,
+        "Rejecting invalid destination address %s",
+        escaped_safe_str(exitconn->base_.address));
+    return -1;
+  }
+
+  /* then take this opportunity to see if there are any expired
+   * resolves in the hash table. */
+  purge_expired_resolves(now);
+
+  /* lower-case exitconn->base_.address, so it's in canonical form */
+  tor_strlower(exitconn->base_.address);
+
+  /* Check whether this is a reverse lookup.  If it's malformed, or it's a
+   * .in-addr.arpa address but this isn't a resolve request, kill the
+   * connection.
+   */
+  if ((r = tor_addr_parse_PTR_name(&addr, exitconn->base_.address,
+                                              AF_UNSPEC, 0)) != 0) {
+    if (r == 1) {
+      is_reverse = 1;
+      if (tor_addr_is_internal(&addr, 0)) /* internal address? */
+        return -1;
+    }
+
+    if (!is_reverse || !is_resolve) {
+      if (!is_reverse)
+        log_info(LD_EXIT, "Bad .in-addr.arpa address \"%s\"; sending error.",
+                 escaped_safe_str(exitconn->base_.address));
+      else if (!is_resolve)
+        log_info(LD_EXIT,
+                 "Attempt to connect to a .in-addr.arpa address \"%s\"; "
+                 "sending error.",
+                 escaped_safe_str(exitconn->base_.address));
+
+      return -1;
+    }
+    //log_notice(LD_EXIT, "Looks like an address %s",
+    //exitconn->base_.address);
+  }
+  exitconn->is_reverse_dns_lookup = is_reverse;
+
+  /* now check the hash table to see if 'address' is already there. */
+  strlcpy(search.address, exitconn->base_.address, sizeof(search.address));
+  resolve = HT_FIND(cache_map, &cache_root, &search);
+  if (resolve && resolve->expire > now) { /* already there */
+    switch (resolve->state) {
+      case CACHE_STATE_PENDING:
+        /* add us to the pending list */
+        pending_connection = tor_malloc_zero(
+                                      sizeof(pending_connection_t));
+        pending_connection->conn = exitconn;
+        pending_connection->next = resolve->pending_connections;
+        resolve->pending_connections = pending_connection;
+        *made_connection_pending_out = 1;
+        log_debug(LD_EXIT,"Connection (fd "TOR_SOCKET_T_FORMAT") waiting "
+                  "for pending DNS resolve of %s", exitconn->base_.s,
+                  escaped_safe_str(exitconn->base_.address));
+        return 0;
+      case CACHE_STATE_CACHED:
+        log_debug(LD_EXIT,"Connection (fd "TOR_SOCKET_T_FORMAT") found "
+                  "cached answer for %s",
+                  exitconn->base_.s,
+                  escaped_safe_str(resolve->address));
+
+        *resolve_out = resolve;
+
+        return set_exitconn_info_from_resolve(exitconn, resolve, hostname_out);
+      case CACHE_STATE_DONE:
+        log_err(LD_BUG, "Found a 'DONE' dns resolve still in the cache.");
+        tor_fragile_assert();
+    }
+    tor_assert(0);
+  }
+  tor_assert(!resolve);
+  /* not there, need to add it */
+  resolve = tor_malloc_zero(sizeof(cached_resolve_t));
+  resolve->magic = CACHED_RESOLVE_MAGIC;
+  resolve->state = CACHE_STATE_PENDING;
+  resolve->minheap_idx = -1;
+  strlcpy(resolve->address, exitconn->base_.address, sizeof(resolve->address));
+
+  /* add this connection to the pending list */
+  pending_connection = tor_malloc_zero(sizeof(pending_connection_t));
+  pending_connection->conn = exitconn;
+  resolve->pending_connections = pending_connection;
+  *made_connection_pending_out = 1;
+
+  /* Add this resolve to the cache and priority queue. */
+  HT_INSERT(cache_map, &cache_root, resolve);
+  set_expiry(resolve, now + RESOLVE_MAX_TIMEOUT);
+
+  log_debug(LD_EXIT,"Launching %s.",
+            escaped_safe_str(exitconn->base_.address));
+  assert_cache_ok();
+
+  return launch_resolve(resolve);
+}
+
+/** Given an exit connection <b>exitconn</b>, and a cached_resolve_t
+ * <b>resolve</b> whose DNS lookups have all either succeeded or failed,
+ * update the appropriate fields (address_ttl and addr) of <b>exitconn</b>.
+ *
+ * The logic can be complicated here, since we might have launched both
+ * an A lookup and an AAAA lookup, and since either of those might have
+ * succeeded or failed, and since we want to answer a RESOLVE cell with
+ * a full answer but answer a BEGIN cell with whatever answer the client
+ * would accept <i>and</i> we could still connect to.
+ *
+ * If this is a reverse lookup, set *<b>hostname_out</b> to a newly allocated
+ * copy of the name resulting hostname.
+ *
+ * Return -2 on a transient error, -1 on a permenent error, and 1 on
+ * a successful lookup.
+ */
+MOCK_IMPL(STATIC int,
+set_exitconn_info_from_resolve,(edge_connection_t *exitconn,
+                                const cached_resolve_t *resolve,
+                                char **hostname_out))
+{
+  int ipv4_ok, ipv6_ok, answer_with_ipv4, r;
+  uint32_t begincell_flags;
+  const int is_resolve = exitconn->base_.purpose == EXIT_PURPOSE_RESOLVE;
+  tor_assert(exitconn);
+  tor_assert(resolve);
+
+  if (exitconn->is_reverse_dns_lookup) {
+    exitconn->address_ttl = resolve->ttl_hostname;
+    if (resolve->res_status_hostname == RES_STATUS_DONE_OK) {
+      *hostname_out = tor_strdup(resolve->result_ptr.hostname);
+      return 1;
+    } else {
+      return -1;
+    }
+  }
+
+  /* If we're here then the connection wants one or either of ipv4, ipv6, and
+   * we can give it one or both. */
+  if (is_resolve) {
+    begincell_flags = BEGIN_FLAG_IPV6_OK;
+  } else {
+    begincell_flags = exitconn->begincell_flags;
+  }
+
+  ipv4_ok = (resolve->res_status_ipv4 == RES_STATUS_DONE_OK) &&
+    ! (begincell_flags & BEGIN_FLAG_IPV4_NOT_OK);
+  ipv6_ok = (resolve->res_status_ipv6 == RES_STATUS_DONE_OK) &&
+    (begincell_flags & BEGIN_FLAG_IPV6_OK) &&
+    get_options()->IPv6Exit;
+
+  /* Now decide which one to actually give. */
+  if (ipv4_ok && ipv6_ok && is_resolve) {
+    answer_with_ipv4 = 1;
+  } else if (ipv4_ok && ipv6_ok) {
+    /* If we have both, see if our exit policy has an opinion. */
+    const uint16_t port = exitconn->base_.port;
+    int ipv4_allowed, ipv6_allowed;
+    tor_addr_t a4, a6;
+    tor_addr_from_ipv4h(&a4, resolve->result_ipv4.addr_ipv4);
+    tor_addr_from_in6(&a6, &resolve->result_ipv6.addr_ipv6);
+    ipv4_allowed = !router_compare_to_my_exit_policy(&a4, port);
+    ipv6_allowed = !router_compare_to_my_exit_policy(&a6, port);
+    if (ipv4_allowed && !ipv6_allowed) {
+      answer_with_ipv4 = 1;
+    } else if (ipv6_allowed && !ipv4_allowed) {
+      answer_with_ipv4 = 0;
+    } else {
+      /* Our exit policy would permit both.  Answer with whichever the user
+       * prefers */
+      answer_with_ipv4 = !(begincell_flags &
+                           BEGIN_FLAG_IPV6_PREFERRED);
+    }
+  } else {
+    /* Otherwise if one is okay, send it back. */
+    if (ipv4_ok) {
+      answer_with_ipv4 = 1;
+    } else if (ipv6_ok) {
+      answer_with_ipv4 = 0;
+    } else {
+      /* Neither one was okay. Choose based on user preference. */
+      answer_with_ipv4 = !(begincell_flags &
+                           BEGIN_FLAG_IPV6_PREFERRED);
+    }
+  }
+
+  /* Finally, we write the answer back. */
+  r = 1;
+  if (answer_with_ipv4) {
+    if (resolve->res_status_ipv4 == RES_STATUS_DONE_OK) {
+      tor_addr_from_ipv4h(&exitconn->base_.addr,
+                          resolve->result_ipv4.addr_ipv4);
+    } else {
+      r = evdns_err_is_transient(resolve->result_ipv4.err_ipv4) ? -2 : -1;
+    }
+
+    exitconn->address_ttl = resolve->ttl_ipv4;
+  } else {
+    if (resolve->res_status_ipv6 == RES_STATUS_DONE_OK) {
+      tor_addr_from_in6(&exitconn->base_.addr,
+                        &resolve->result_ipv6.addr_ipv6);
+    } else {
+      r = evdns_err_is_transient(resolve->result_ipv6.err_ipv6) ? -2 : -1;
+    }
+
+    exitconn->address_ttl = resolve->ttl_ipv6;
+  }
+
+  return r;
+}
+
+/** Log an error and abort if conn is waiting for a DNS resolve.
+ */
+void
+assert_connection_edge_not_dns_pending(edge_connection_t *conn)
+{
+  pending_connection_t *pend;
+  cached_resolve_t search;
+
+#if 1
+  cached_resolve_t *resolve;
+  strlcpy(search.address, conn->base_.address, sizeof(search.address));
+  resolve = HT_FIND(cache_map, &cache_root, &search);
+  if (!resolve)
+    return;
+  for (pend = resolve->pending_connections; pend; pend = pend->next) {
+    tor_assert(pend->conn != conn);
+  }
+#else /* !(1) */
+  cached_resolve_t **resolve;
+  HT_FOREACH(resolve, cache_map, &cache_root) {
+    for (pend = (*resolve)->pending_connections; pend; pend = pend->next) {
+      tor_assert(pend->conn != conn);
+    }
+  }
+#endif /* 1 */
+}
+
+/** Log an error and abort if any connection waiting for a DNS resolve is
+ * corrupted. */
+void
+assert_all_pending_dns_resolves_ok(void)
+{
+  pending_connection_t *pend;
+  cached_resolve_t **resolve;
+
+  HT_FOREACH(resolve, cache_map, &cache_root) {
+    for (pend = (*resolve)->pending_connections;
+         pend;
+         pend = pend->next) {
+      assert_connection_ok(TO_CONN(pend->conn), 0);
+      tor_assert(!SOCKET_OK(pend->conn->base_.s));
+      tor_assert(!connection_in_array(TO_CONN(pend->conn)));
+    }
+  }
+}
+
+/** Remove <b>conn</b> from the list of connections waiting for conn-\>address.
+ */
+void
+connection_dns_remove(edge_connection_t *conn)
+{
+  pending_connection_t *pend, *victim;
+  cached_resolve_t search;
+  cached_resolve_t *resolve;
+
+  tor_assert(conn->base_.type == CONN_TYPE_EXIT);
+  tor_assert(conn->base_.state == EXIT_CONN_STATE_RESOLVING);
+
+  strlcpy(search.address, conn->base_.address, sizeof(search.address));
+
+  resolve = HT_FIND(cache_map, &cache_root, &search);
+  if (!resolve) {
+    log_notice(LD_BUG, "Address %s is not pending. Dropping.",
+               escaped_safe_str(conn->base_.address));
+    return;
+  }
+
+  tor_assert(resolve->pending_connections);
+  assert_connection_ok(TO_CONN(conn),0);
+
+  pend = resolve->pending_connections;
+
+  if (pend->conn == conn) {
+    resolve->pending_connections = pend->next;
+    tor_free(pend);
+    log_debug(LD_EXIT, "First connection (fd "TOR_SOCKET_T_FORMAT") no "
+              "longer waiting for resolve of %s",
+              conn->base_.s,
+              escaped_safe_str(conn->base_.address));
+    return;
+  } else {
+    for ( ; pend->next; pend = pend->next) {
+      if (pend->next->conn == conn) {
+        victim = pend->next;
+        pend->next = victim->next;
+        tor_free(victim);
+        log_debug(LD_EXIT,
+                  "Connection (fd "TOR_SOCKET_T_FORMAT") no longer waiting "
+                  "for resolve of %s",
+                  conn->base_.s, escaped_safe_str(conn->base_.address));
+        return; /* more are pending */
+      }
+    }
+    log_warn(LD_BUG, "Connection (fd "TOR_SOCKET_T_FORMAT") was not waiting "
+             "for a resolve of %s, but we tried to remove it.",
+             conn->base_.s, escaped_safe_str(conn->base_.address));
+  }
+}
+
+/** Mark all connections waiting for <b>address</b> for close.  Then cancel
+ * the resolve for <b>address</b> itself, and remove any cached results for
+ * <b>address</b> from the cache.
+ */
+MOCK_IMPL(void,
+dns_cancel_pending_resolve,(const char *address))
+{
+  pending_connection_t *pend;
+  cached_resolve_t search;
+  cached_resolve_t *resolve, *tmp;
+  edge_connection_t *pendconn;
+  circuit_t *circ;
+
+  strlcpy(search.address, address, sizeof(search.address));
+
+  resolve = HT_FIND(cache_map, &cache_root, &search);
+  if (!resolve)
+    return;
+
+  if (resolve->state != CACHE_STATE_PENDING) {
+    /* We can get into this state if we never actually created the pending
+     * resolve, due to finding an earlier cached error or something.  Just
+     * ignore it. */
+    if (resolve->pending_connections) {
+      log_warn(LD_BUG,
+               "Address %s is not pending but has pending connections!",
+               escaped_safe_str(address));
+      tor_fragile_assert();
+    }
+    return;
+  }
+
+  if (!resolve->pending_connections) {
+    log_warn(LD_BUG,
+             "Address %s is pending but has no pending connections!",
+             escaped_safe_str(address));
+    tor_fragile_assert();
+    return;
+  }
+  tor_assert(resolve->pending_connections);
+
+  /* mark all pending connections to fail */
+  log_debug(LD_EXIT,
+             "Failing all connections waiting on DNS resolve of %s",
+             escaped_safe_str(address));
+  while (resolve->pending_connections) {
+    pend = resolve->pending_connections;
+    pend->conn->base_.state = EXIT_CONN_STATE_RESOLVEFAILED;
+    pendconn = pend->conn;
+    assert_connection_ok(TO_CONN(pendconn), 0);
+    tor_assert(!SOCKET_OK(pendconn->base_.s));
+    if (!pendconn->base_.marked_for_close) {
+      connection_edge_end(pendconn, END_STREAM_REASON_RESOLVEFAILED);
+    }
+    circ = circuit_get_by_edge_conn(pendconn);
+    if (circ)
+      circuit_detach_stream(circ, pendconn);
+    if (!pendconn->base_.marked_for_close)
+      connection_free_(TO_CONN(pendconn));
+    resolve->pending_connections = pend->next;
+    tor_free(pend);
+  }
+
+  tmp = HT_REMOVE(cache_map, &cache_root, resolve);
+  if (tmp != resolve) {
+    log_err(LD_BUG, "The cancelled resolve we purged didn't match any in"
+            " the cache. Tried to purge %s (%p); instead got %s (%p).",
+            resolve->address, (void*)resolve,
+            tmp ? tmp->address : "NULL", (void*)tmp);
+  }
+  tor_assert(tmp == resolve);
+
+  resolve->state = CACHE_STATE_DONE;
+}
+
+/** Return true iff <b>address</b> is one of the addresses we use to verify
+ * that well-known sites aren't being hijacked by our DNS servers. */
+static inline int
+is_test_address(const char *address)
+{
+  const or_options_t *options = get_options();
+  return options->ServerDNSTestAddresses &&
+    smartlist_contains_string_case(options->ServerDNSTestAddresses, address);
+}
+
+/** Called on the OR side when the eventdns library tells us the outcome of a
+ * single DNS resolve: remember the answer, and tell all pending connections
+ * about the result of the lookup if the lookup is now done.  (<b>address</b>
+ * is a NUL-terminated string containing the address to look up;
+ * <b>query_type</b> is one of DNS_{IPv4_A,IPv6_AAAA,PTR}; <b>dns_answer</b>
+ * is DNS_OK or one of DNS_ERR_*, <b>addr</b> is an IPv4 or IPv6 address if we
+ * got one; <b>hostname</b> is a hostname fora PTR request if we got one, and
+ * <b>ttl</b> is the time-to-live of this answer, in seconds.)
+ */
+static void
+dns_found_answer(const char *address, uint8_t query_type,
+                 int dns_answer,
+                 const tor_addr_t *addr,
+                 const char *hostname, uint32_t ttl)
+{
+  cached_resolve_t search;
+  cached_resolve_t *resolve;
+
+  assert_cache_ok();
+
+  strlcpy(search.address, address, sizeof(search.address));
+
+  resolve = HT_FIND(cache_map, &cache_root, &search);
+  if (!resolve) {
+    int is_test_addr = is_test_address(address);
+    if (!is_test_addr)
+      log_info(LD_EXIT,"Resolved unasked address %s; ignoring.",
+               escaped_safe_str(address));
+    return;
+  }
+  assert_resolve_ok(resolve);
+
+  if (resolve->state != CACHE_STATE_PENDING) {
+    /* XXXX Maybe update addr? or check addr for consistency? Or let
+     * VALID replace FAILED? */
+    int is_test_addr = is_test_address(address);
+    if (!is_test_addr)
+      log_notice(LD_EXIT,
+                 "Resolved %s which was already resolved; ignoring",
+                 escaped_safe_str(address));
+    tor_assert(resolve->pending_connections == NULL);
+    return;
+  }
+
+  cached_resolve_add_answer(resolve, query_type, dns_answer,
+                            addr, hostname, ttl);
+
+  if (cached_resolve_have_all_answers(resolve)) {
+    inform_pending_connections(resolve);
+
+    make_pending_resolve_cached(resolve);
+  }
+}
+
+/** Given a pending cached_resolve_t that we just finished resolving,
+ * inform every connection that was waiting for the outcome of that
+ * resolution.
+ *
+ * Do this by sending a RELAY_RESOLVED cell (if the pending stream had sent us
+ * RELAY_RESOLVE cell), or by launching an exit connection (if the pending
+ * stream had send us a RELAY_BEGIN cell).
+ */
+static void
+inform_pending_connections(cached_resolve_t *resolve)
+{
+  pending_connection_t *pend;
+  edge_connection_t *pendconn;
+  int r;
+
+  while (resolve->pending_connections) {
+    char *hostname = NULL;
+    pend = resolve->pending_connections;
+    pendconn = pend->conn; /* don't pass complex things to the
+                              connection_mark_for_close macro */
+    assert_connection_ok(TO_CONN(pendconn),time(NULL));
+
+    if (pendconn->base_.marked_for_close) {
+      /* prevent double-remove. */
+      pendconn->base_.state = EXIT_CONN_STATE_RESOLVEFAILED;
+      resolve->pending_connections = pend->next;
+      tor_free(pend);
+      continue;
+    }
+
+    r = set_exitconn_info_from_resolve(pendconn,
+                                       resolve,
+                                       &hostname);
+
+    if (r < 0) {
+      /* prevent double-remove. */
+      pendconn->base_.state = EXIT_CONN_STATE_RESOLVEFAILED;
+      if (pendconn->base_.purpose == EXIT_PURPOSE_CONNECT) {
+        connection_edge_end(pendconn, END_STREAM_REASON_RESOLVEFAILED);
+        /* This detach must happen after we send the end cell. */
+        circuit_detach_stream(circuit_get_by_edge_conn(pendconn), pendconn);
+      } else {
+        send_resolved_cell(pendconn, r == -1 ?
+                         RESOLVED_TYPE_ERROR : RESOLVED_TYPE_ERROR_TRANSIENT,
+                         NULL);
+        /* This detach must happen after we send the resolved cell. */
+        circuit_detach_stream(circuit_get_by_edge_conn(pendconn), pendconn);
+      }
+      connection_free_(TO_CONN(pendconn));
+    } else {
+      circuit_t *circ;
+      if (pendconn->base_.purpose == EXIT_PURPOSE_CONNECT) {
+        /* prevent double-remove. */
+        pend->conn->base_.state = EXIT_CONN_STATE_CONNECTING;
+
+        circ = circuit_get_by_edge_conn(pend->conn);
+        tor_assert(circ);
+        tor_assert(!CIRCUIT_IS_ORIGIN(circ));
+        /* unlink pend->conn from resolving_streams, */
+        circuit_detach_stream(circ, pend->conn);
+        /* and link it to n_streams */
+        pend->conn->next_stream = TO_OR_CIRCUIT(circ)->n_streams;
+        pend->conn->on_circuit = circ;
+        TO_OR_CIRCUIT(circ)->n_streams = pend->conn;
+
+        connection_exit_connect(pend->conn);
+      } else {
+        /* prevent double-remove.  This isn't really an accurate state,
+         * but it does the right thing. */
+        pendconn->base_.state = EXIT_CONN_STATE_RESOLVEFAILED;
+        if (pendconn->is_reverse_dns_lookup)
+          send_resolved_hostname_cell(pendconn, hostname);
+        else
+          send_resolved_cell(pendconn, RESOLVED_TYPE_AUTO, resolve);
+        circ = circuit_get_by_edge_conn(pendconn);
+        tor_assert(circ);
+        circuit_detach_stream(circ, pendconn);
+        connection_free_(TO_CONN(pendconn));
+      }
+    }
+    resolve->pending_connections = pend->next;
+    tor_free(pend);
+    tor_free(hostname);
+  }
+}
+
+/** Remove a pending cached_resolve_t from the hashtable, and add a
+ * corresponding cached cached_resolve_t.
+ *
+ * This function is only necessary because of the perversity of our
+ * cache timeout code; see inline comment for ideas on eliminating it.
+ **/
+static void
+make_pending_resolve_cached(cached_resolve_t *resolve)
+{
+  cached_resolve_t *removed;
+
+  resolve->state = CACHE_STATE_DONE;
+  removed = HT_REMOVE(cache_map, &cache_root, resolve);
+  if (removed != resolve) {
+    log_err(LD_BUG, "The pending resolve we found wasn't removable from"
+            " the cache. Tried to purge %s (%p); instead got %s (%p).",
+            resolve->address, (void*)resolve,
+            removed ? removed->address : "NULL", (void*)removed);
+  }
+  assert_resolve_ok(resolve);
+  assert_cache_ok();
+  /* The resolve will eventually just hit the time-out in the expiry queue and
+  * expire. See fd0bafb0dedc7e2 for a brief explanation of how this got that
+  * way.  XXXXX we could do better!*/
+
+  {
+    cached_resolve_t *new_resolve = tor_memdup(resolve,
+                                               sizeof(cached_resolve_t));
+    uint32_t ttl = UINT32_MAX;
+    new_resolve->expire = 0; /* So that set_expiry won't croak. */
+    if (resolve->res_status_hostname == RES_STATUS_DONE_OK)
+      new_resolve->result_ptr.hostname =
+        tor_strdup(resolve->result_ptr.hostname);
+
+    new_resolve->state = CACHE_STATE_CACHED;
+
+    assert_resolve_ok(new_resolve);
+    HT_INSERT(cache_map, &cache_root, new_resolve);
+
+    if ((resolve->res_status_ipv4 == RES_STATUS_DONE_OK ||
+         resolve->res_status_ipv4 == RES_STATUS_DONE_ERR) &&
+        resolve->ttl_ipv4 < ttl)
+      ttl = resolve->ttl_ipv4;
+
+    if ((resolve->res_status_ipv6 == RES_STATUS_DONE_OK ||
+         resolve->res_status_ipv6 == RES_STATUS_DONE_ERR) &&
+        resolve->ttl_ipv6 < ttl)
+      ttl = resolve->ttl_ipv6;
+
+    if ((resolve->res_status_hostname == RES_STATUS_DONE_OK ||
+         resolve->res_status_hostname == RES_STATUS_DONE_ERR) &&
+        resolve->ttl_hostname < ttl)
+      ttl = resolve->ttl_hostname;
+
+    set_expiry(new_resolve, time(NULL) + dns_clip_ttl(ttl));
+  }
+
+  assert_cache_ok();
+}
+
+/** Eventdns helper: return true iff the eventdns result <b>err</b> is
+ * a transient failure. */
+static int
+evdns_err_is_transient(int err)
+{
+  switch (err)
+  {
+    case DNS_ERR_SERVERFAILED:
+    case DNS_ERR_TRUNCATED:
+    case DNS_ERR_TIMEOUT:
+      return 1;
+    default:
+      return 0;
+  }
+}
+
+/** Configure eventdns nameservers if force is true, or if the configuration
+ * has changed since the last time we called this function, or if we failed on
+ * our last attempt.  On Unix, this reads from /etc/resolv.conf or
+ * options->ServerDNSResolvConfFile; on Windows, this reads from
+ * options->ServerDNSResolvConfFile or the registry.  Return 0 on success or
+ * -1 on failure. */
+static int
+configure_nameservers(int force)
+{
+  const or_options_t *options;
+  const char *conf_fname;
+  struct stat st;
+  int r, flags;
+  options = get_options();
+  conf_fname = options->ServerDNSResolvConfFile;
+#ifndef _WIN32
+  if (!conf_fname)
+    conf_fname = "/etc/resolv.conf";
+#endif
+  flags = DNS_OPTIONS_ALL;
+
+  if (!the_evdns_base) {
+    if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) {
+      log_err(LD_BUG, "Couldn't create an evdns_base");
+      return -1;
+    }
+  }
+
+  evdns_set_log_fn(evdns_log_cb);
+  if (conf_fname) {
+    log_debug(LD_FS, "stat()ing %s", conf_fname);
+    if (stat(sandbox_intern_string(conf_fname), &st)) {
+      log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s",
+               conf_fname, strerror(errno));
+      goto err;
+    }
+    if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname)
+        && st.st_mtime == resolv_conf_mtime) {
+      log_info(LD_EXIT, "No change to '%s'", conf_fname);
+      return 0;
+    }
+    if (nameservers_configured) {
+      evdns_base_search_clear(the_evdns_base);
+      evdns_base_clear_nameservers_and_suspend(the_evdns_base);
+    }
+#if defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP)
+    if (flags & DNS_OPTION_HOSTSFILE) {
+      flags ^= DNS_OPTION_HOSTSFILE;
+      log_debug(LD_FS, "Loading /etc/hosts");
+      evdns_base_load_hosts(the_evdns_base,
+          sandbox_intern_string("/etc/hosts"));
+    }
+#endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */
+    log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
+    if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
+        sandbox_intern_string(conf_fname)))) {
+      log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)",
+               conf_fname, conf_fname, r);
+      goto err;
+    }
+    if (evdns_base_count_nameservers(the_evdns_base) == 0) {
+      log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname);
+      goto err;
+    }
+    tor_free(resolv_conf_fname);
+    resolv_conf_fname = tor_strdup(conf_fname);
+    resolv_conf_mtime = st.st_mtime;
+    if (nameservers_configured)
+      evdns_base_resume(the_evdns_base);
+  }
+#ifdef _WIN32
+  else {
+    if (nameservers_configured) {
+      evdns_base_search_clear(the_evdns_base);
+      evdns_base_clear_nameservers_and_suspend(the_evdns_base);
+    }
+    if (evdns_base_config_windows_nameservers(the_evdns_base))  {
+      log_warn(LD_EXIT,"Could not config nameservers.");
+      goto err;
+    }
+    if (evdns_base_count_nameservers(the_evdns_base) == 0) {
+      log_warn(LD_EXIT, "Unable to find any platform nameservers in "
+               "your Windows configuration.");
+      goto err;
+    }
+    if (nameservers_configured)
+      evdns_base_resume(the_evdns_base);
+    tor_free(resolv_conf_fname);
+    resolv_conf_mtime = 0;
+  }
+#endif /* defined(_WIN32) */
+
+#define SET(k,v)  evdns_base_set_option(the_evdns_base, (k), (v))
+
+  // If we only have one nameserver, it does not make sense to back off
+  // from it for a timeout. Unfortunately, the value for max-timeouts is
+  // currently clamped by libevent to 255, but it does not hurt to set
+  // it higher in case libevent gets a patch for this.  Higher-than-
+  // default maximum of 3 with multiple nameservers to avoid spuriously
+  // marking one down on bursts of timeouts resulting from scans/attacks
+  // against non-responding authoritative DNS servers.
+  if (evdns_base_count_nameservers(the_evdns_base) == 1) {
+    SET("max-timeouts:", "1000000");
+  } else {
+    SET("max-timeouts:", "10");
+  }
+
+  // Elongate the queue of maximum inflight dns requests, so if a bunch
+  // remain pending at the resolver (happens commonly with Unbound) we won't
+  // stall every other DNS request. This potentially means some wasted
+  // CPU as there's a walk over a linear queue involved, but this is a
+  // much better tradeoff compared to just failing DNS requests because
+  // of a full queue.
+  SET("max-inflight:", "8192");
+
+  // Two retries at 5 and 10 seconds for bind9/named which relies on
+  // clients to handle retries.  Second retry for retried circuits with
+  // extended 15 second timeout.  Superfluous with local-system Unbound
+  // instance--has its own elaborate retry scheme.
+  SET("timeout:", "5");
+  SET("attempts:","3");
+
+  if (options->ServerDNSRandomizeCase)
+    SET("randomize-case:", "1");
+  else
+    SET("randomize-case:", "0");
+
+#undef SET
+
+  dns_servers_relaunch_checks();
+
+  nameservers_configured = 1;
+  if (nameserver_config_failed) {
+    nameserver_config_failed = 0;
+    /* XXX the three calls to republish the descriptor might be producing
+     * descriptors that are only cosmetically different, especially on
+     * non-exit relays! -RD */
+    mark_my_descriptor_dirty("dns resolvers back");
+  }
+  return 0;
+ err:
+  nameservers_configured = 0;
+  if (! nameserver_config_failed) {
+    nameserver_config_failed = 1;
+    mark_my_descriptor_dirty("dns resolvers failed");
+  }
+  return -1;
+}
+
+/** For eventdns: Called when we get an answer for a request we launched.
+ * See eventdns.h for arguments; 'arg' holds the address we tried to resolve.
+ */
+static void
+evdns_callback(int result, char type, int count, int ttl, void *addresses,
+               void *arg)
+{
+  char *arg_ = arg;
+  uint8_t orig_query_type = arg_[0];
+  char *string_address = arg_ + 1;
+  tor_addr_t addr;
+  const char *hostname = NULL;
+  int was_wildcarded = 0;
+
+  tor_addr_make_unspec(&addr);
+
+  /* Keep track of whether IPv6 is working */
+  if (type == DNS_IPv6_AAAA) {
+    if (result == DNS_ERR_TIMEOUT) {
+      ++n_ipv6_timeouts;
+    }
+
+    if (n_ipv6_timeouts > 10 &&
+        n_ipv6_timeouts > n_ipv6_requests_made / 2) {
+      if (! dns_is_broken_for_ipv6) {
+        log_notice(LD_EXIT, "More than half of our IPv6 requests seem to "
+                   "have timed out. I'm going to assume I can't get AAAA "
+                   "responses.");
+        dns_is_broken_for_ipv6 = 1;
+      }
+    }
+  }
+
+  if (result == DNS_ERR_NONE) {
+    if (type == DNS_IPv4_A && count) {
+      char answer_buf[INET_NTOA_BUF_LEN+1];
+      char *escaped_address;
+      uint32_t *addrs = addresses;
+      tor_addr_from_ipv4n(&addr, addrs[0]);
+
+      tor_addr_to_str(answer_buf, &addr, sizeof(answer_buf), 0);
+      escaped_address = esc_for_log(string_address);
+
+      if (answer_is_wildcarded(answer_buf)) {
+        log_debug(LD_EXIT, "eventdns said that %s resolves to ISP-hijacked "
+                  "address %s; treating as a failure.",
+                  safe_str(escaped_address),
+                  escaped_safe_str(answer_buf));
+        was_wildcarded = 1;
+        tor_addr_make_unspec(&addr);
+        result = DNS_ERR_NOTEXIST;
+      } else {
+        log_debug(LD_EXIT, "eventdns said that %s resolves to %s",
+                  safe_str(escaped_address),
+                  escaped_safe_str(answer_buf));
+      }
+      tor_free(escaped_address);
+    } else if (type == DNS_IPv6_AAAA && count) {
+      char answer_buf[TOR_ADDR_BUF_LEN];
+      char *escaped_address;
+      struct in6_addr *addrs = addresses;
+      tor_addr_from_in6(&addr, &addrs[0]);
+      tor_inet_ntop(AF_INET6, &addrs[0], answer_buf, sizeof(answer_buf));
+      escaped_address = esc_for_log(string_address);
+
+      if (answer_is_wildcarded(answer_buf)) {
+        log_debug(LD_EXIT, "eventdns said that %s resolves to ISP-hijacked "
+                  "address %s; treating as a failure.",
+                  safe_str(escaped_address),
+                  escaped_safe_str(answer_buf));
+        was_wildcarded = 1;
+        tor_addr_make_unspec(&addr);
+        result = DNS_ERR_NOTEXIST;
+      } else {
+        log_debug(LD_EXIT, "eventdns said that %s resolves to %s",
+                  safe_str(escaped_address),
+                  escaped_safe_str(answer_buf));
+      }
+      tor_free(escaped_address);
+    } else if (type == DNS_PTR && count) {
+      char *escaped_address;
+      hostname = ((char**)addresses)[0];
+      escaped_address = esc_for_log(string_address);
+      log_debug(LD_EXIT, "eventdns said that %s resolves to %s",
+                safe_str(escaped_address),
+                escaped_safe_str(hostname));
+      tor_free(escaped_address);
+    } else if (count) {
+      log_info(LD_EXIT, "eventdns returned only unrecognized answer types "
+               " for %s.",
+               escaped_safe_str(string_address));
+    } else {
+      log_info(LD_EXIT, "eventdns returned no addresses or error for %s.",
+               escaped_safe_str(string_address));
+    }
+  }
+  if (was_wildcarded) {
+    if (is_test_address(string_address)) {
+      /* Ick.  We're getting redirected on known-good addresses.  Our DNS
+       * server must really hate us.  */
+      add_wildcarded_test_address(string_address);
+    }
+  }
+
+  if (orig_query_type && type && orig_query_type != type) {
+    log_warn(LD_BUG, "Weird; orig_query_type == %d but type == %d",
+             (int)orig_query_type, (int)type);
+  }
+  if (result != DNS_ERR_SHUTDOWN)
+    dns_found_answer(string_address, orig_query_type,
+                     result, &addr, hostname, ttl);
+
+  tor_free(arg_);
+}
+
+/** Start a single DNS resolve for <b>address</b> (if <b>query_type</b> is
+ * DNS_IPv4_A or DNS_IPv6_AAAA) <b>ptr_address</b> (if <b>query_type</b> is
+ * DNS_PTR). Return 0 if we launched the request, -1 otherwise. */
+static int
+launch_one_resolve(const char *address, uint8_t query_type,
+                   const tor_addr_t *ptr_address)
+{
+  const int options = get_options()->ServerDNSSearchDomains ? 0
+    : DNS_QUERY_NO_SEARCH;
+  const size_t addr_len = strlen(address);
+  struct evdns_request *req = 0;
+  char *addr = tor_malloc(addr_len + 2);
+  addr[0] = (char) query_type;
+  memcpy(addr+1, address, addr_len + 1);
+
+  switch (query_type) {
+  case DNS_IPv4_A:
+    req = evdns_base_resolve_ipv4(the_evdns_base,
+                                  address, options, evdns_callback, addr);
+    break;
+  case DNS_IPv6_AAAA:
+    req = evdns_base_resolve_ipv6(the_evdns_base,
+                                  address, options, evdns_callback, addr);
+    ++n_ipv6_requests_made;
+    break;
+  case DNS_PTR:
+    if (tor_addr_family(ptr_address) == AF_INET)
+      req = evdns_base_resolve_reverse(the_evdns_base,
+                                       tor_addr_to_in(ptr_address),
+                                       DNS_QUERY_NO_SEARCH,
+                                       evdns_callback, addr);
+    else if (tor_addr_family(ptr_address) == AF_INET6)
+      req = evdns_base_resolve_reverse_ipv6(the_evdns_base,
+                                            tor_addr_to_in6(ptr_address),
+                                            DNS_QUERY_NO_SEARCH,
+                                            evdns_callback, addr);
+    else
+      log_warn(LD_BUG, "Called with PTR query and unexpected address family");
+    break;
+  default:
+    log_warn(LD_BUG, "Called with unexpectd query type %d", (int)query_type);
+    break;
+  }
+
+  if (req) {
+    return 0;
+  } else {
+    tor_free(addr);
+    return -1;
+  }
+}
+
+/** For eventdns: start resolving as necessary to find the target for
+ * <b>exitconn</b>.  Returns -1 on error, -2 on transient error,
+ * 0 on "resolve launched." */
+MOCK_IMPL(STATIC int,
+launch_resolve,(cached_resolve_t *resolve))
+{
+  tor_addr_t a;
+  int r;
+
+  if (net_is_disabled())
+    return -1;
+
+  /* What? Nameservers not configured?  Sounds like a bug. */
+  if (!nameservers_configured) {
+    log_warn(LD_EXIT, "(Harmless.) Nameservers not configured, but resolve "
+             "launched.  Configuring.");
+    if (configure_nameservers(1) < 0) {
+      return -1;
+    }
+  }
+
+  r = tor_addr_parse_PTR_name(
+                            &a, resolve->address, AF_UNSPEC, 0);
+
+  tor_assert(the_evdns_base);
+  if (r == 0) {
+    log_info(LD_EXIT, "Launching eventdns request for %s",
+             escaped_safe_str(resolve->address));
+    resolve->res_status_ipv4 = RES_STATUS_INFLIGHT;
+    if (get_options()->IPv6Exit)
+      resolve->res_status_ipv6 = RES_STATUS_INFLIGHT;
+
+    if (launch_one_resolve(resolve->address, DNS_IPv4_A, NULL) < 0) {
+      resolve->res_status_ipv4 = 0;
+      r = -1;
+    }
+
+    if (r==0 && get_options()->IPv6Exit) {
+      /* We ask for an IPv6 address for *everything*. */
+      if (launch_one_resolve(resolve->address, DNS_IPv6_AAAA, NULL) < 0) {
+        resolve->res_status_ipv6 = 0;
+        r = -1;
+      }
+    }
+  } else if (r == 1) {
+    r = 0;
+    log_info(LD_EXIT, "Launching eventdns reverse request for %s",
+             escaped_safe_str(resolve->address));
+    resolve->res_status_hostname = RES_STATUS_INFLIGHT;
+    if (launch_one_resolve(resolve->address, DNS_PTR, &a) < 0) {
+      resolve->res_status_hostname = 0;
+      r = -1;
+    }
+  } else if (r == -1) {
+    log_warn(LD_BUG, "Somehow a malformed in-addr.arpa address reached here.");
+  }
+
+  if (r < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_EXIT, "eventdns rejected address %s.",
+           escaped_safe_str(resolve->address));
+  }
+  return r;
+}
+
+/** How many requests for bogus addresses have we launched so far? */
+static int n_wildcard_requests = 0;
+
+/** Map from dotted-quad IP address in response to an int holding how many
+ * times we've seen it for a randomly generated (hopefully bogus) address.  It
+ * would be easier to use definitely-invalid addresses (as specified by
+ * RFC2606), but see comment in dns_launch_wildcard_checks(). */
+static strmap_t *dns_wildcard_response_count = NULL;
+
+/** If present, a list of dotted-quad IP addresses that we are pretty sure our
+ * nameserver wants to return in response to requests for nonexistent domains.
+ */
+static smartlist_t *dns_wildcard_list = NULL;
+/** True iff we've logged about a single address getting wildcarded.
+ * Subsequent warnings will be less severe.  */
+static int dns_wildcard_one_notice_given = 0;
+/** True iff we've warned that our DNS server is wildcarding too many failures.
+ */
+static int dns_wildcard_notice_given = 0;
+
+/** List of supposedly good addresses that are getting wildcarded to the
+ * same addresses as nonexistent addresses. */
+static smartlist_t *dns_wildcarded_test_address_list = NULL;
+/** True iff we've warned about a test address getting wildcarded */
+static int dns_wildcarded_test_address_notice_given = 0;
+/** True iff all addresses seem to be getting wildcarded. */
+static int dns_is_completely_invalid = 0;
+
+/** Called when we see <b>id</b> (a dotted quad or IPv6 address) in response
+ * to a request for a hopefully bogus address. */
+static void
+wildcard_increment_answer(const char *id)
+{
+  int *ip;
+  if (!dns_wildcard_response_count)
+    dns_wildcard_response_count = strmap_new();
+
+  ip = strmap_get(dns_wildcard_response_count, id); // may be null (0)
+  if (!ip) {
+    ip = tor_malloc_zero(sizeof(int));
+    strmap_set(dns_wildcard_response_count, id, ip);
+  }
+  ++*ip;
+
+  if (*ip > 5 && n_wildcard_requests > 10) {
+    if (!dns_wildcard_list) dns_wildcard_list = smartlist_new();
+    if (!smartlist_contains_string(dns_wildcard_list, id)) {
+    tor_log(dns_wildcard_notice_given ? LOG_INFO : LOG_NOTICE, LD_EXIT,
+        "Your DNS provider has given \"%s\" as an answer for %d different "
+        "invalid addresses. Apparently they are hijacking DNS failures. "
+        "I'll try to correct for this by treating future occurrences of "
+        "\"%s\" as 'not found'.", id, *ip, id);
+      smartlist_add_strdup(dns_wildcard_list, id);
+    }
+    if (!dns_wildcard_notice_given)
+      control_event_server_status(LOG_NOTICE, "DNS_HIJACKED");
+    dns_wildcard_notice_given = 1;
+  }
+}
+
+/** Note that a single test address (one believed to be good) seems to be
+ * getting redirected to the same IP as failures are. */
+static void
+add_wildcarded_test_address(const char *address)
+{
+  int n, n_test_addrs;
+  if (!dns_wildcarded_test_address_list)
+    dns_wildcarded_test_address_list = smartlist_new();
+
+  if (smartlist_contains_string_case(dns_wildcarded_test_address_list,
+                                     address))
+    return;
+
+  n_test_addrs = get_options()->ServerDNSTestAddresses ?
+    smartlist_len(get_options()->ServerDNSTestAddresses) : 0;
+
+  smartlist_add_strdup(dns_wildcarded_test_address_list, address);
+  n = smartlist_len(dns_wildcarded_test_address_list);
+  if (n > n_test_addrs/2) {
+    tor_log(dns_wildcarded_test_address_notice_given ? LOG_INFO : LOG_NOTICE,
+        LD_EXIT, "Your DNS provider tried to redirect \"%s\" to a junk "
+        "address.  It has done this with %d test addresses so far.  I'm "
+        "going to stop being an exit node for now, since our DNS seems so "
+        "broken.", address, n);
+    if (!dns_is_completely_invalid) {
+      dns_is_completely_invalid = 1;
+      mark_my_descriptor_dirty("dns hijacking confirmed");
+    }
+    if (!dns_wildcarded_test_address_notice_given)
+      control_event_server_status(LOG_WARN, "DNS_USELESS");
+    dns_wildcarded_test_address_notice_given = 1;
+  }
+}
+
+/** Callback function when we get an answer (possibly failing) for a request
+ * for a (hopefully) nonexistent domain. */
+static void
+evdns_wildcard_check_callback(int result, char type, int count, int ttl,
+                              void *addresses, void *arg)
+{
+  (void)ttl;
+  ++n_wildcard_requests;
+  if (result == DNS_ERR_NONE && count) {
+    char *string_address = arg;
+    int i;
+    if (type == DNS_IPv4_A) {
+      const uint32_t *addrs = addresses;
+      for (i = 0; i < count; ++i) {
+        char answer_buf[INET_NTOA_BUF_LEN+1];
+        struct in_addr in;
+        in.s_addr = addrs[i];
+        tor_inet_ntoa(&in, answer_buf, sizeof(answer_buf));
+        wildcard_increment_answer(answer_buf);
+      }
+    } else if (type == DNS_IPv6_AAAA) {
+      const struct in6_addr *addrs = addresses;
+      for (i = 0; i < count; ++i) {
+        char answer_buf[TOR_ADDR_BUF_LEN+1];
+        tor_inet_ntop(AF_INET6, &addrs[i], answer_buf, sizeof(answer_buf));
+        wildcard_increment_answer(answer_buf);
+      }
+    }
+
+    tor_log(dns_wildcard_one_notice_given ? LOG_INFO : LOG_NOTICE, LD_EXIT,
+        "Your DNS provider gave an answer for \"%s\", which "
+        "is not supposed to exist. Apparently they are hijacking "
+        "DNS failures. Trying to correct for this. We've noticed %d "
+        "possibly bad address%s so far.",
+        string_address, strmap_size(dns_wildcard_response_count),
+        (strmap_size(dns_wildcard_response_count) == 1) ? "" : "es");
+    dns_wildcard_one_notice_given = 1;
+  }
+  tor_free(arg);
+}
+
+/** Launch a single request for a nonexistent hostname consisting of between
+ * <b>min_len</b> and <b>max_len</b> random (plausible) characters followed by
+ * <b>suffix</b> */
+static void
+launch_wildcard_check(int min_len, int max_len, int is_ipv6,
+                      const char *suffix)
+{
+  char *addr;
+  struct evdns_request *req;
+
+  addr = crypto_random_hostname(min_len, max_len, "", suffix);
+  log_info(LD_EXIT, "Testing whether our DNS server is hijacking nonexistent "
+           "domains with request for bogus hostname \"%s\"", addr);
+
+  tor_assert(the_evdns_base);
+  if (is_ipv6)
+    req = evdns_base_resolve_ipv6(
+                         the_evdns_base,
+                         /* This "addr" tells us which address to resolve */
+                         addr,
+                         DNS_QUERY_NO_SEARCH, evdns_wildcard_check_callback,
+                         /* This "addr" is an argument to the callback*/ addr);
+  else
+    req = evdns_base_resolve_ipv4(
+                         the_evdns_base,
+                         /* This "addr" tells us which address to resolve */
+                         addr,
+                         DNS_QUERY_NO_SEARCH, evdns_wildcard_check_callback,
+                         /* This "addr" is an argument to the callback*/ addr);
+  if (!req) {
+    /* There is no evdns request in progress; stop addr from getting leaked */
+    tor_free(addr);
+  }
+}
+
+/** Launch attempts to resolve a bunch of known-good addresses (configured in
+ * ServerDNSTestAddresses).  [Callback for a libevent timer] */
+static void
+launch_test_addresses(evutil_socket_t fd, short event, void *args)
+{
+  const or_options_t *options = get_options();
+  (void)fd;
+  (void)event;
+  (void)args;
+
+  if (net_is_disabled())
+    return;
+
+  log_info(LD_EXIT, "Launching checks to see whether our nameservers like to "
+           "hijack *everything*.");
+  /* This situation is worse than the failure-hijacking situation.  When this
+   * happens, we're no good for DNS requests at all, and we shouldn't really
+   * be an exit server.*/
+  if (options->ServerDNSTestAddresses) {
+
+    tor_assert(the_evdns_base);
+    SMARTLIST_FOREACH_BEGIN(options->ServerDNSTestAddresses,
+                            const char *, address) {
+      if (launch_one_resolve(address, DNS_IPv4_A, NULL) < 0) {
+        log_info(LD_EXIT, "eventdns rejected test address %s",
+                 escaped_safe_str(address));
+      }
+
+      if (launch_one_resolve(address, DNS_IPv6_AAAA, NULL) < 0) {
+        log_info(LD_EXIT, "eventdns rejected test address %s",
+                 escaped_safe_str(address));
+      }
+    } SMARTLIST_FOREACH_END(address);
+  }
+}
+
+#define N_WILDCARD_CHECKS 2
+
+/** Launch DNS requests for a few nonexistent hostnames and a few well-known
+ * hostnames, and see if we can catch our nameserver trying to hijack them and
+ * map them to a stupid "I couldn't find ggoogle.com but maybe you'd like to
+ * buy these lovely encyclopedias" page. */
+static void
+dns_launch_wildcard_checks(void)
+{
+  int i, ipv6;
+  log_info(LD_EXIT, "Launching checks to see whether our nameservers like "
+           "to hijack DNS failures.");
+  for (ipv6 = 0; ipv6 <= 1; ++ipv6) {
+    for (i = 0; i < N_WILDCARD_CHECKS; ++i) {
+      /* RFC2606 reserves these.  Sadly, some DNS hijackers, in a silly
+       * attempt to 'comply' with rfc2606, refrain from giving A records for
+       * these.  This is the standards-compliance equivalent of making sure
+       * that your crackhouse's elevator inspection certificate is up to date.
+       */
+      launch_wildcard_check(2, 16, ipv6, ".invalid");
+      launch_wildcard_check(2, 16, ipv6, ".test");
+
+      /* These will break specs if there are ever any number of
+       * 8+-character top-level domains. */
+      launch_wildcard_check(8, 16, ipv6, "");
+
+      /* Try some random .com/org/net domains. This will work fine so long as
+       * not too many resolve to the same place. */
+      launch_wildcard_check(8, 16, ipv6, ".com");
+      launch_wildcard_check(8, 16, ipv6, ".org");
+      launch_wildcard_check(8, 16, ipv6, ".net");
+    }
+  }
+}
+
+/** If appropriate, start testing whether our DNS servers tend to lie to
+ * us. */
+void
+dns_launch_correctness_checks(void)
+{
+  static struct event *launch_event = NULL;
+  struct timeval timeout;
+  if (!get_options()->ServerDNSDetectHijacking)
+    return;
+  dns_launch_wildcard_checks();
+
+  /* Wait a while before launching requests for test addresses, so we can
+   * get the results from checking for wildcarding. */
+  if (! launch_event)
+    launch_event = tor_evtimer_new(tor_libevent_get_base(),
+                                   launch_test_addresses, NULL);
+  timeout.tv_sec = 30;
+  timeout.tv_usec = 0;
+  if (evtimer_add(launch_event, &timeout)<0) {
+    log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking");
+  }
+}
+
+/** Return true iff our DNS servers lie to us too much to be trusted. */
+int
+dns_seems_to_be_broken(void)
+{
+  return dns_is_completely_invalid;
+}
+
+/** Return true iff we think that IPv6 hostname lookup is broken */
+int
+dns_seems_to_be_broken_for_ipv6(void)
+{
+  return dns_is_broken_for_ipv6;
+}
+
+/** Forget what we've previously learned about our DNS servers' correctness. */
+void
+dns_reset_correctness_checks(void)
+{
+  strmap_free(dns_wildcard_response_count, tor_free_);
+  dns_wildcard_response_count = NULL;
+
+  n_wildcard_requests = 0;
+
+  n_ipv6_requests_made = n_ipv6_timeouts = 0;
+
+  if (dns_wildcard_list) {
+    SMARTLIST_FOREACH(dns_wildcard_list, char *, cp, tor_free(cp));
+    smartlist_clear(dns_wildcard_list);
+  }
+  if (dns_wildcarded_test_address_list) {
+    SMARTLIST_FOREACH(dns_wildcarded_test_address_list, char *, cp,
+                      tor_free(cp));
+    smartlist_clear(dns_wildcarded_test_address_list);
+  }
+  dns_wildcard_one_notice_given = dns_wildcard_notice_given =
+    dns_wildcarded_test_address_notice_given = dns_is_completely_invalid =
+    dns_is_broken_for_ipv6 = 0;
+}
+
+/** Return true iff we have noticed that the dotted-quad <b>ip</b> has been
+ * returned in response to requests for nonexistent hostnames. */
+static int
+answer_is_wildcarded(const char *ip)
+{
+  return dns_wildcard_list && smartlist_contains_string(dns_wildcard_list, ip);
+}
+
+/** Exit with an assertion if <b>resolve</b> is corrupt. */
+static void
+assert_resolve_ok(cached_resolve_t *resolve)
+{
+  tor_assert(resolve);
+  tor_assert(resolve->magic == CACHED_RESOLVE_MAGIC);
+  tor_assert(strlen(resolve->address) < MAX_ADDRESSLEN);
+  tor_assert(tor_strisnonupper(resolve->address));
+  if (resolve->state != CACHE_STATE_PENDING) {
+    tor_assert(!resolve->pending_connections);
+  }
+  if (resolve->state == CACHE_STATE_PENDING ||
+      resolve->state == CACHE_STATE_DONE) {
+#if 0
+    tor_assert(!resolve->ttl);
+    if (resolve->is_reverse)
+      tor_assert(!resolve->hostname);
+    else
+      tor_assert(!resolve->result_ipv4.addr_ipv4);
+#endif /* 0 */
+    /*XXXXX ADD MORE */
+  }
+}
+
+/** Return the number of DNS cache entries as an int */
+static int
+dns_cache_entry_count(void)
+{
+   return HT_SIZE(&cache_root);
+}
+
+/** Log memory information about our internal DNS cache at level 'severity'. */
+void
+dump_dns_mem_usage(int severity)
+{
+  /* This should never be larger than INT_MAX. */
+  int hash_count = dns_cache_entry_count();
+  size_t hash_mem = sizeof(struct cached_resolve_t) * hash_count;
+  hash_mem += HT_MEM_USAGE(&cache_root);
+
+  /* Print out the count and estimated size of our &cache_root.  It undercounts
+     hostnames in cached reverse resolves.
+   */
+  tor_log(severity, LD_MM, "Our DNS cache has %d entries.", hash_count);
+  tor_log(severity, LD_MM, "Our DNS cache size is approximately %u bytes.",
+      (unsigned)hash_mem);
+}
+
+#ifdef DEBUG_DNS_CACHE
+/** Exit with an assertion if the DNS cache is corrupt. */
+static void
+assert_cache_ok_(void)
+{
+  cached_resolve_t **resolve;
+  int bad_rep = HT_REP_IS_BAD_(cache_map, &cache_root);
+  if (bad_rep) {
+    log_err(LD_BUG, "Bad rep type %d on dns cache hash table", bad_rep);
+    tor_assert(!bad_rep);
+  }
+
+  HT_FOREACH(resolve, cache_map, &cache_root) {
+    assert_resolve_ok(*resolve);
+    tor_assert((*resolve)->state != CACHE_STATE_DONE);
+  }
+  if (!cached_resolve_pqueue)
+    return;
+
+  smartlist_pqueue_assert_ok(cached_resolve_pqueue,
+                             compare_cached_resolves_by_expiry_,
+                             offsetof(cached_resolve_t, minheap_idx));
+
+  SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res,
+    {
+      if (res->state == CACHE_STATE_DONE) {
+        cached_resolve_t *found = HT_FIND(cache_map, &cache_root, res);
+        tor_assert(!found || found != res);
+      } else {
+        cached_resolve_t *found = HT_FIND(cache_map, &cache_root, res);
+        tor_assert(found);
+      }
+    });
+}
+
+#endif /* defined(DEBUG_DNS_CACHE) */
+
+cached_resolve_t *
+dns_get_cache_entry(cached_resolve_t *query)
+{
+  return HT_FIND(cache_map, &cache_root, query);
+}
+
+void
+dns_insert_cache_entry(cached_resolve_t *new_entry)
+{
+  HT_INSERT(cache_map, &cache_root, new_entry);
+}
diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h
new file mode 100644
index 0000000000..12853205ff
--- /dev/null
+++ b/src/feature/relay/dns.h
@@ -0,0 +1,70 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dns.h
+ * \brief Header file for dns.c.
+ **/
+
+#ifndef TOR_DNS_H
+#define TOR_DNS_H
+
+/** Lowest value for DNS ttl that a server will give. */
+#define MIN_DNS_TTL_AT_EXIT (5*60)
+/** Highest value for DNS ttl that a server will give. */
+#define MAX_DNS_TTL_AT_EXIT (60*60)
+
+/** How long do we keep DNS cache entries before purging them (regardless of
+ * their TTL)? */
+#define MAX_DNS_ENTRY_AGE (3*60*60)
+/** How long do we cache/tell clients to cache DNS records when no TTL is
+ * known? */
+#define DEFAULT_DNS_TTL (30*60)
+
+int dns_init(void);
+int has_dns_init_failed(void);
+void dns_free_all(void);
+uint32_t dns_clip_ttl(uint32_t ttl);
+int dns_reset(void);
+void connection_dns_remove(edge_connection_t *conn);
+void assert_connection_edge_not_dns_pending(edge_connection_t *conn);
+void assert_all_pending_dns_resolves_ok(void);
+MOCK_DECL(void,dns_cancel_pending_resolve,(const char *question));
+int dns_resolve(edge_connection_t *exitconn);
+void dns_launch_correctness_checks(void);
+int dns_seems_to_be_broken(void);
+int dns_seems_to_be_broken_for_ipv6(void);
+void dns_reset_correctness_checks(void);
+void dump_dns_mem_usage(int severity);
+
+#ifdef DNS_PRIVATE
+#include "or/dns_structs.h"
+
+MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn,
+int is_resolve,or_circuit_t *oncirc, char **hostname_out,
+int *made_connection_pending_out, cached_resolve_t **resolve_out));
+
+MOCK_DECL(STATIC void,send_resolved_cell,(edge_connection_t *conn,
+uint8_t answer_type,const cached_resolve_t *resolved));
+
+MOCK_DECL(STATIC void,send_resolved_hostname_cell,(edge_connection_t *conn,
+const char *hostname));
+
+cached_resolve_t *dns_get_cache_entry(cached_resolve_t *query);
+void dns_insert_cache_entry(cached_resolve_t *new_entry);
+
+MOCK_DECL(STATIC int,
+set_exitconn_info_from_resolve,(edge_connection_t *exitconn,
+                                const cached_resolve_t *resolve,
+                                char **hostname_out));
+
+MOCK_DECL(STATIC int,
+launch_resolve,(cached_resolve_t *resolve));
+
+#endif /* defined(DNS_PRIVATE) */
+
+#endif /* !defined(TOR_DNS_H) */
+
diff --git a/src/feature/relay/dns_structs.h b/src/feature/relay/dns_structs.h
new file mode 100644
index 0000000000..28c48ca0bc
--- /dev/null
+++ b/src/feature/relay/dns_structs.h
@@ -0,0 +1,102 @@
+/* Copyright (c) 2003-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dns_structs.h
+ *
+ * \brief Structures used in dns.c. Exposed to dns.c, and to the unit tests
+ * that declare DNS_PRIVATE.
+ */
+
+#ifndef TOR_DNS_STRUCTS_H
+#define TOR_DNS_STRUCTS_H
+
+/** Longest hostname we're willing to resolve. */
+#define MAX_ADDRESSLEN 256
+
+/** Linked list of connections waiting for a DNS answer. */
+typedef struct pending_connection_t {
+  edge_connection_t *conn;
+  struct pending_connection_t *next;
+} pending_connection_t;
+
+/** Value of 'magic' field for cached_resolve_t.  Used to try to catch bad
+ * pointers and memory stomping. */
+#define CACHED_RESOLVE_MAGIC 0x1234F00D
+
+/* Possible states for a cached resolve_t */
+/** We are waiting for the resolver system to tell us an answer here.
+ * When we get one, or when we time out, the state of this cached_resolve_t
+ * will become "DONE" and we'll possibly add a CACHED
+ * entry. This cached_resolve_t will be in the hash table so that we will
+ * know not to launch more requests for this addr, but rather to add more
+ * connections to the pending list for the addr. */
+#define CACHE_STATE_PENDING 0
+/** This used to be a pending cached_resolve_t, and we got an answer for it.
+ * Now we're waiting for this cached_resolve_t to expire.  This should
+ * have no pending connections, and should not appear in the hash table. */
+#define CACHE_STATE_DONE 1
+/** We are caching an answer for this address. This should have no pending
+ * connections, and should appear in the hash table. */
+#define CACHE_STATE_CACHED 2
+
+/** @name status values for a single DNS request.
+ *
+ * @{ */
+/** The DNS request is in progress. */
+#define RES_STATUS_INFLIGHT 1
+/** The DNS request finished and gave an answer */
+#define RES_STATUS_DONE_OK 2
+/** The DNS request finished and gave an error */
+#define RES_STATUS_DONE_ERR 3
+/**@}*/
+
+/** A DNS request: possibly completed, possibly pending; cached_resolve
+ * structs are stored at the OR side in a hash table, and as a linked
+ * list from oldest to newest.
+ */
+typedef struct cached_resolve_t {
+  HT_ENTRY(cached_resolve_t) node;
+  uint32_t magic;  /**< Must be CACHED_RESOLVE_MAGIC */
+  char address[MAX_ADDRESSLEN]; /**< The hostname to be resolved. */
+
+  union {
+    uint32_t addr_ipv4; /**< IPv4 addr for <b>address</b>, if successful.
+                         * (In host order.) */
+    int err_ipv4; /**< One of DNS_ERR_*, if IPv4 lookup failed. */
+  } result_ipv4; /**< Outcome of IPv4 lookup */
+  union {
+    struct in6_addr addr_ipv6; /**< IPv6 addr for <b>address</b>, if
+                                * successful */
+    int err_ipv6; /**< One of DNS_ERR_*, if IPv6 lookup failed. */
+  } result_ipv6; /**< Outcome of IPv6 lookup, if any */
+  union {
+    char *hostname; /** A hostname, if PTR lookup happened successfully*/
+    int err_hostname; /** One of DNS_ERR_*, if PTR lookup failed. */
+  } result_ptr;
+  /** @name Status fields
+   *
+   * These take one of the RES_STATUS_* values, depending on the state
+   * of the corresponding lookup.
+   *
+   * @{ */
+  unsigned int res_status_ipv4 : 2;
+  unsigned int res_status_ipv6 : 2;
+  unsigned int res_status_hostname : 2;
+  /**@}*/
+  uint8_t state; /**< Is this cached entry pending/done/informative? */
+
+  time_t expire; /**< Remove items from cache after this time. */
+  uint32_t ttl_ipv4; /**< What TTL did the nameserver tell us? */
+  uint32_t ttl_ipv6; /**< What TTL did the nameserver tell us? */
+  uint32_t ttl_hostname; /**< What TTL did the nameserver tell us? */
+  /** Connections that want to know when we get an answer for this resolve. */
+  pending_connection_t *pending_connections;
+  /** Position of this element in the heap*/
+  int minheap_idx;
+} cached_resolve_t;
+
+#endif /* !defined(TOR_DNS_STRUCTS_H) */
+
diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c
new file mode 100644
index 0000000000..7342a66e06
--- /dev/null
+++ b/src/feature/relay/ext_orport.c
@@ -0,0 +1,662 @@
+/* Copyright (c) 2012-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ext_orport.c
+ * \brief Code implementing the Extended ORPort.
+ *
+ * The Extended ORPort interface is used by pluggable transports to
+ * communicate additional information to a Tor bridge, including
+ * address information. For more information on this interface,
+ * see pt-spec.txt in torspec.git.
+ *
+ * There is no separate structure for extended ORPort connections; they use
+ * or_connection_t objects, and share most of their implementation with
+ * connection_or.c.  Once the handshake is done, an extended ORPort connection
+ * turns into a regular OR connection, using connection_ext_or_transition().
+ */
+
+#define EXT_ORPORT_PRIVATE
+#include "or/or.h"
+#include "or/connection.h"
+#include "or/connection_or.h"
+#include "or/control.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/ext_orport.h"
+#include "or/main.h"
+#include "or/proto_ext_or.h"
+
+#include "or/or_connection_st.h"
+
+/** Allocate and return a structure capable of holding an Extended
+ *  ORPort message of body length <b>len</b>. */
+ext_or_cmd_t *
+ext_or_cmd_new(uint16_t len)
+{
+  size_t size = offsetof(ext_or_cmd_t, body) + len;
+  ext_or_cmd_t *cmd = tor_malloc(size);
+  cmd->len = len;
+  return cmd;
+}
+
+/** Deallocate the Extended ORPort message in <b>cmd</b>. */
+void
+ext_or_cmd_free_(ext_or_cmd_t *cmd)
+{
+  tor_free(cmd);
+}
+
+/** Get an Extended ORPort message from <b>conn</b>, and place it in
+ *  <b>out</b>. Return -1 on fail, 0 if we need more data, and 1 if we
+ *  successfully extracted an Extended ORPort command from the
+ *  buffer.  */
+static int
+connection_fetch_ext_or_cmd_from_buf(connection_t *conn, ext_or_cmd_t **out)
+{
+  return fetch_ext_or_command_from_buf(conn->inbuf, out);
+}
+
+/** Write an Extended ORPort message to <b>conn</b>. Use
+ *  <b>command</b> as the command type, <b>bodylen</b> as the body
+ *  length, and <b>body</b>, if it's present, as the body of the
+ *  message. */
+STATIC int
+connection_write_ext_or_command(connection_t *conn,
+                                uint16_t command,
+                                const char *body,
+                                size_t bodylen)
+{
+  char header[4];
+  if (bodylen > UINT16_MAX)
+    return -1;
+  set_uint16(header, htons(command));
+  set_uint16(header+2, htons(bodylen));
+  connection_buf_add(header, 4, conn);
+  if (bodylen) {
+    tor_assert(body);
+    connection_buf_add(body, bodylen, conn);
+  }
+  return 0;
+}
+
+/** Transition from an Extended ORPort which accepts Extended ORPort
+ *  messages, to an Extended ORport which accepts OR traffic. */
+static void
+connection_ext_or_transition(or_connection_t *conn)
+{
+  tor_assert(conn->base_.type == CONN_TYPE_EXT_OR);
+
+  conn->base_.type = CONN_TYPE_OR;
+  TO_CONN(conn)->state = 0; // set the state to a neutral value
+  control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0);
+  connection_tls_start_handshake(conn, 1);
+}
+
+/** Length of authentication cookie. */
+#define EXT_OR_PORT_AUTH_COOKIE_LEN 32
+/** Length of the header of the cookie file. */
+#define EXT_OR_PORT_AUTH_COOKIE_HEADER_LEN 32
+/** Static cookie file header. */
+#define EXT_OR_PORT_AUTH_COOKIE_HEADER "! Extended ORPort Auth Cookie !\x0a"
+/** Length of safe-cookie protocol hashes. */
+#define EXT_OR_PORT_AUTH_HASH_LEN DIGEST256_LEN
+/** Length of safe-cookie protocol nonces. */
+#define EXT_OR_PORT_AUTH_NONCE_LEN 32
+/** Safe-cookie protocol constants. */
+#define EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST \
+  "ExtORPort authentication server-to-client hash"
+#define EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST \
+  "ExtORPort authentication client-to-server hash"
+
+/* Code to indicate cookie authentication */
+#define EXT_OR_AUTHTYPE_SAFECOOKIE 0x01
+
+/** If true, we've set ext_or_auth_cookie to a secret code and stored
+ * it to disk. */
+STATIC int ext_or_auth_cookie_is_set = 0;
+/** If ext_or_auth_cookie_is_set, a secret cookie that we've stored to disk
+ * and which we're using to authenticate controllers.  (If the controller can
+ * read it off disk, it has permission to connect.) */
+STATIC uint8_t *ext_or_auth_cookie = NULL;
+
+/** Helper: Return a newly allocated string containing a path to the
+ * file where we store our authentication cookie. */
+char *
+get_ext_or_auth_cookie_file_name(void)
+{
+  const or_options_t *options = get_options();
+  if (options->ExtORPortCookieAuthFile &&
+      strlen(options->ExtORPortCookieAuthFile)) {
+    return tor_strdup(options->ExtORPortCookieAuthFile);
+  } else {
+    return get_datadir_fname("extended_orport_auth_cookie");
+  }
+}
+
+/* Initialize the cookie-based authentication system of the
+ * Extended ORPort. If <b>is_enabled</b> is 0, then disable the cookie
+ * authentication system. */
+int
+init_ext_or_cookie_authentication(int is_enabled)
+{
+  char *fname = NULL;
+  int retval;
+
+  if (!is_enabled) {
+    ext_or_auth_cookie_is_set = 0;
+    return 0;
+  }
+
+  fname = get_ext_or_auth_cookie_file_name();
+  retval = init_cookie_authentication(fname, EXT_OR_PORT_AUTH_COOKIE_HEADER,
+                                      EXT_OR_PORT_AUTH_COOKIE_HEADER_LEN,
+                           get_options()->ExtORPortCookieAuthFileGroupReadable,
+                                      &ext_or_auth_cookie,
+                                      &ext_or_auth_cookie_is_set);
+  tor_free(fname);
+  return retval;
+}
+
+/** Read data from <b>conn</b> and see if the client sent us the
+ *  authentication type that they prefer to use in this session.
+ *
+ *  Return -1 if we received corrupted data or if we don't support the
+ *  authentication type. Return 0 if we need more data in
+ *  <b>conn</b>. Return 1 if the authentication type negotiation was
+ *  successful. */
+static int
+connection_ext_or_auth_neg_auth_type(connection_t *conn)
+{
+  char authtype[1] = {0};
+
+  if (connection_get_inbuf_len(conn) < 1)
+    return 0;
+
+  if (connection_buf_get_bytes(authtype, 1, conn) < 0)
+    return -1;
+
+  log_debug(LD_GENERAL, "Client wants us to use %d auth type", authtype[0]);
+  if (authtype[0] != EXT_OR_AUTHTYPE_SAFECOOKIE) {
+    /* '1' is the only auth type supported atm */
+    return -1;
+  }
+
+  conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE;
+  return 1;
+}
+
+/* DOCDOC */
+STATIC int
+handle_client_auth_nonce(const char *client_nonce, size_t client_nonce_len,
+                         char **client_hash_out,
+                         char **reply_out, size_t *reply_len_out)
+{
+  char server_hash[EXT_OR_PORT_AUTH_HASH_LEN] = {0};
+  char server_nonce[EXT_OR_PORT_AUTH_NONCE_LEN] = {0};
+  char *reply;
+  size_t reply_len;
+
+  if (client_nonce_len != EXT_OR_PORT_AUTH_NONCE_LEN)
+    return -1;
+
+  /* Get our nonce */
+  crypto_rand(server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN);
+
+  { /* set up macs */
+    size_t hmac_s_msg_len = strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST) +
+      2*EXT_OR_PORT_AUTH_NONCE_LEN;
+    size_t hmac_c_msg_len = strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST) +
+      2*EXT_OR_PORT_AUTH_NONCE_LEN;
+
+    char *hmac_s_msg = tor_malloc_zero(hmac_s_msg_len);
+    char *hmac_c_msg = tor_malloc_zero(hmac_c_msg_len);
+    char *correct_client_hash = tor_malloc_zero(EXT_OR_PORT_AUTH_HASH_LEN);
+
+    memcpy(hmac_s_msg,
+           EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST,
+           strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST));
+    memcpy(hmac_s_msg + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST),
+           client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN);
+    memcpy(hmac_s_msg + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST) +
+           EXT_OR_PORT_AUTH_NONCE_LEN,
+           server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN);
+
+    memcpy(hmac_c_msg,
+           EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST,
+           strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST));
+    memcpy(hmac_c_msg + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST),
+           client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN);
+    memcpy(hmac_c_msg + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST) +
+           EXT_OR_PORT_AUTH_NONCE_LEN,
+           server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN);
+
+    crypto_hmac_sha256(server_hash,
+                       (char*)ext_or_auth_cookie,
+                       EXT_OR_PORT_AUTH_COOKIE_LEN,
+                       hmac_s_msg,
+                       hmac_s_msg_len);
+
+    crypto_hmac_sha256(correct_client_hash,
+                       (char*)ext_or_auth_cookie,
+                       EXT_OR_PORT_AUTH_COOKIE_LEN,
+                       hmac_c_msg,
+                       hmac_c_msg_len);
+
+    /* Store the client hash we generated. We will need to compare it
+       with the hash sent by the client. */
+    *client_hash_out = correct_client_hash;
+
+    memwipe(hmac_s_msg, 0, hmac_s_msg_len);
+    memwipe(hmac_c_msg, 0, hmac_c_msg_len);
+
+    tor_free(hmac_s_msg);
+    tor_free(hmac_c_msg);
+  }
+
+  { /* debug logging */ /* XXX disable this codepath if not logging on debug?*/
+    char server_hash_encoded[(2*EXT_OR_PORT_AUTH_HASH_LEN) + 1];
+    char server_nonce_encoded[(2*EXT_OR_PORT_AUTH_NONCE_LEN) + 1];
+    char client_nonce_encoded[(2*EXT_OR_PORT_AUTH_NONCE_LEN) + 1];
+
+    base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
+                  server_hash, sizeof(server_hash));
+    base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
+                  server_nonce, sizeof(server_nonce));
+    base16_encode(client_nonce_encoded, sizeof(client_nonce_encoded),
+                  client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN);
+
+    log_debug(LD_GENERAL,
+              "server_hash: '%s'\nserver_nonce: '%s'\nclient_nonce: '%s'",
+              server_hash_encoded, server_nonce_encoded, client_nonce_encoded);
+
+    memwipe(server_hash_encoded, 0, sizeof(server_hash_encoded));
+    memwipe(server_nonce_encoded, 0, sizeof(server_nonce_encoded));
+    memwipe(client_nonce_encoded, 0, sizeof(client_nonce_encoded));
+  }
+
+  { /* write reply: (server_hash, server_nonce) */
+
+    reply_len = EXT_OR_PORT_AUTH_COOKIE_LEN+EXT_OR_PORT_AUTH_NONCE_LEN;
+    reply = tor_malloc_zero(reply_len);
+    memcpy(reply, server_hash, EXT_OR_PORT_AUTH_HASH_LEN);
+    memcpy(reply + EXT_OR_PORT_AUTH_HASH_LEN, server_nonce,
+           EXT_OR_PORT_AUTH_NONCE_LEN);
+  }
+
+  *reply_out = reply;
+  *reply_len_out = reply_len;
+
+  return 0;
+}
+
+/** Read the client's nonce out of <b>conn</b>, setup the safe-cookie
+ *  crypto, and then send our own hash and nonce to the client
+ *
+ *  Return -1 if there was an error; return 0 if we need more data in
+ *  <b>conn</b>, and return 1 if we successfully retrieved the
+ *  client's nonce and sent our own. */
+static int
+connection_ext_or_auth_handle_client_nonce(connection_t *conn)
+{
+  char client_nonce[EXT_OR_PORT_AUTH_NONCE_LEN];
+  char *reply=NULL;
+  size_t reply_len=0;
+
+  if (!ext_or_auth_cookie_is_set) { /* this should not happen */
+    log_warn(LD_BUG, "Extended ORPort authentication cookie was not set. "
+             "That's weird since we should have done that on startup. "
+             "This might be a Tor bug, please file a bug report. ");
+    return -1;
+  }
+
+  if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_NONCE_LEN)
+    return 0;
+
+  if (connection_buf_get_bytes(client_nonce,
+                                EXT_OR_PORT_AUTH_NONCE_LEN, conn) < 0)
+    return -1;
+
+  /* We extract the ClientNonce from the received data, and use it to
+     calculate ServerHash and ServerNonce according to proposal 217.
+
+     We also calculate our own ClientHash value and save it in the
+     connection state. We validate it later against the ClientHash
+     sent by the client.  */
+  if (handle_client_auth_nonce(client_nonce, sizeof(client_nonce),
+                            &TO_OR_CONN(conn)->ext_or_auth_correct_client_hash,
+                            &reply, &reply_len) < 0)
+    return -1;
+
+  connection_buf_add(reply, reply_len, conn);
+
+  memwipe(reply, 0, reply_len);
+  tor_free(reply);
+
+  log_debug(LD_GENERAL, "Got client nonce, and sent our own nonce and hash.");
+
+  conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH;
+  return 1;
+}
+
+#define connection_ext_or_auth_send_result_success(c)  \
+  connection_ext_or_auth_send_result(c, 1)
+#define connection_ext_or_auth_send_result_fail(c)  \
+  connection_ext_or_auth_send_result(c, 0)
+
+/** Send authentication results to <b>conn</b>. Successful results if
+ *  <b>success</b> is set; failure results otherwise. */
+static void
+connection_ext_or_auth_send_result(connection_t *conn, int success)
+{
+  if (success)
+    connection_buf_add("\x01", 1, conn);
+  else
+    connection_buf_add("\x00", 1, conn);
+}
+
+/** Receive the client's hash from <b>conn</b>, validate that it's
+ *  correct, and then send the authentication results to the client.
+ *
+ *  Return -1 if there was an error during validation; return 0 if we
+ *  need more data in <b>conn</b>, and return 1 if we successfully
+ *  validated the client's hash and sent a happy authentication
+ *  result. */
+static int
+connection_ext_or_auth_handle_client_hash(connection_t *conn)
+{
+  char provided_client_hash[EXT_OR_PORT_AUTH_HASH_LEN] = {0};
+
+  if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_HASH_LEN)
+    return 0;
+
+  if (connection_buf_get_bytes(provided_client_hash,
+                                EXT_OR_PORT_AUTH_HASH_LEN, conn) < 0)
+    return -1;
+
+  if (tor_memneq(TO_OR_CONN(conn)->ext_or_auth_correct_client_hash,
+                 provided_client_hash, EXT_OR_PORT_AUTH_HASH_LEN)) {
+    log_warn(LD_GENERAL, "Incorrect client hash. Authentication failed.");
+    connection_ext_or_auth_send_result_fail(conn);
+    return -1;
+  }
+
+  log_debug(LD_GENERAL, "Got client's hash and it was legit.");
+
+  /* send positive auth result */
+  connection_ext_or_auth_send_result_success(conn);
+  conn->state = EXT_OR_CONN_STATE_OPEN;
+  return 1;
+}
+
+/** Handle data from <b>or_conn</b> received on Extended ORPort.
+ *  Return -1 on error. 0 on unsufficient data. 1 on correct. */
+static int
+connection_ext_or_auth_process_inbuf(or_connection_t *or_conn)
+{
+  connection_t *conn = TO_CONN(or_conn);
+
+  /* State transitions of the Extended ORPort authentication protocol:
+
+     EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE (start state) ->
+     EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE ->
+     EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH ->
+     EXT_OR_CONN_STATE_OPEN
+
+     During EXT_OR_CONN_STATE_OPEN, data is handled by
+     connection_ext_or_process_inbuf().
+  */
+
+  switch (conn->state) { /* Functionify */
+  case EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE:
+    return connection_ext_or_auth_neg_auth_type(conn);
+
+  case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE:
+    return connection_ext_or_auth_handle_client_nonce(conn);
+
+  case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH:
+    return connection_ext_or_auth_handle_client_hash(conn);
+
+  default:
+    log_warn(LD_BUG, "Encountered unexpected connection state %d while trying "
+             "to process Extended ORPort authentication data.", conn->state);
+    return -1;
+  }
+}
+
+/** Extended ORPort commands (Transport-to-Bridge) */
+#define EXT_OR_CMD_TB_DONE 0x0000
+#define EXT_OR_CMD_TB_USERADDR 0x0001
+#define EXT_OR_CMD_TB_TRANSPORT 0x0002
+
+/** Extended ORPort commands (Bridge-to-Transport) */
+#define EXT_OR_CMD_BT_OKAY 0x1000
+#define EXT_OR_CMD_BT_DENY 0x1001
+#define EXT_OR_CMD_BT_CONTROL 0x1002
+
+/** Process a USERADDR command from the Extended
+ *  ORPort. <b>payload</b> is a payload of size <b>len</b>.
+ *
+ *  If the USERADDR command was well formed, change the address of
+ *  <b>conn</b> to the address on the USERADDR command.
+ *
+ *  Return 0 on success and -1 on error. */
+static int
+connection_ext_or_handle_cmd_useraddr(connection_t *conn,
+                                      const char *payload, uint16_t len)
+{
+  /* Copy address string. */
+  tor_addr_t addr;
+  uint16_t port;
+  char *addr_str;
+  char *address_part=NULL;
+  int res;
+  if (memchr(payload, '\0', len)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unexpected NUL in ExtORPort UserAddr");
+    return -1;
+  }
+
+  addr_str = tor_memdup_nulterm(payload, len);
+
+  res = tor_addr_port_split(LOG_INFO, addr_str, &address_part, &port);
+  tor_free(addr_str);
+  if (res<0)
+    return -1;
+  if (port == 0) {
+    log_warn(LD_GENERAL, "Server transport proxy gave us an empty port "
+             "in ExtORPort UserAddr command.");
+    // return -1; // enable this if nothing breaks after a while.
+  }
+
+  res = tor_addr_parse(&addr, address_part);
+  tor_free(address_part);
+  if (res<0)
+    return -1;
+
+  { /* do some logging */
+    char *old_address = tor_addr_to_str_dup(&conn->addr);
+    char *new_address = tor_addr_to_str_dup(&addr);
+
+    log_debug(LD_NET, "Received USERADDR."
+             "We rewrite our address from '%s:%u' to '%s:%u'.",
+             safe_str(old_address), conn->port, safe_str(new_address), port);
+
+    tor_free(old_address);
+    tor_free(new_address);
+  }
+
+  /* record the address */
+  tor_addr_copy(&conn->addr, &addr);
+  conn->port = port;
+  if (conn->address) {
+    tor_free(conn->address);
+  }
+  conn->address = tor_addr_to_str_dup(&addr);
+
+  return 0;
+}
+
+/** Process a TRANSPORT command from the Extended
+ *  ORPort. <b>payload</b> is a payload of size <b>len</b>.
+ *
+ *  If the TRANSPORT command was well formed, register the name of the
+ *  transport on <b>conn</b>.
+ *
+ *  Return 0 on success and -1 on error. */
+static int
+connection_ext_or_handle_cmd_transport(or_connection_t *conn,
+                                       const char *payload, uint16_t len)
+{
+  char *transport_str;
+  if (memchr(payload, '\0', len)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unexpected NUL in ExtORPort Transport");
+    return -1;
+  }
+
+  transport_str = tor_memdup_nulterm(payload, len);
+
+  /* Transport names MUST be C-identifiers. */
+  if (!string_is_C_identifier(transport_str)) {
+    tor_free(transport_str);
+    return -1;
+  }
+
+  /* If ext_or_transport is already occupied (because the PT sent two
+   *  TRANSPORT commands), deallocate the old name and keep the new
+   *  one */
+  if (conn->ext_or_transport)
+    tor_free(conn->ext_or_transport);
+
+  conn->ext_or_transport = transport_str;
+  return 0;
+}
+
+#define EXT_OR_CONN_STATE_IS_AUTHENTICATING(st) \
+  ((st) <= EXT_OR_CONN_STATE_AUTH_MAX)
+
+/** Process Extended ORPort messages from <b>or_conn</b>. */
+int
+connection_ext_or_process_inbuf(or_connection_t *or_conn)
+{
+  connection_t *conn = TO_CONN(or_conn);
+  ext_or_cmd_t *command;
+  int r;
+
+  /* DOCDOC Document the state machine and transitions in this function */
+
+  /* If we are still in the authentication stage, process traffic as
+     authentication data: */
+  while (EXT_OR_CONN_STATE_IS_AUTHENTICATING(conn->state)) {
+    log_debug(LD_GENERAL, "Got Extended ORPort authentication data (%u).",
+              (unsigned int) connection_get_inbuf_len(conn));
+    r = connection_ext_or_auth_process_inbuf(or_conn);
+    if (r < 0) {
+      connection_mark_for_close(conn);
+      return -1;
+    } else if (r == 0) {
+      return 0;
+    }
+    /* if r > 0, loop and process more data (if any). */
+  }
+
+  while (1) {
+    log_debug(LD_GENERAL, "Got Extended ORPort data.");
+    command = NULL;
+    r = connection_fetch_ext_or_cmd_from_buf(conn, &command);
+    if (r < 0)
+      goto err;
+    else if (r == 0)
+      return 0; /* need to wait for more data */
+
+    /* Got a command! */
+    tor_assert(command);
+
+    if (command->cmd == EXT_OR_CMD_TB_DONE) {
+      if (connection_get_inbuf_len(conn)) {
+        /* The inbuf isn't empty; the client is misbehaving. */
+        goto err;
+      }
+
+      log_debug(LD_NET, "Received DONE.");
+
+      /* If the transport proxy did not use the TRANSPORT command to
+       * specify the transport name, mark this as unknown transport. */
+      if (!or_conn->ext_or_transport) {
+        /* We write this string this way to avoid ??>, which is a C
+         * trigraph. */
+        or_conn->ext_or_transport = tor_strdup("<?" "?>");
+      }
+
+      connection_write_ext_or_command(conn, EXT_OR_CMD_BT_OKAY, NULL, 0);
+
+      /* can't transition immediately; need to flush first. */
+      conn->state = EXT_OR_CONN_STATE_FLUSHING;
+      connection_stop_reading(conn);
+    } else if (command->cmd == EXT_OR_CMD_TB_USERADDR) {
+      if (connection_ext_or_handle_cmd_useraddr(conn,
+                                            command->body, command->len) < 0)
+        goto err;
+    } else if (command->cmd == EXT_OR_CMD_TB_TRANSPORT) {
+      if (connection_ext_or_handle_cmd_transport(or_conn,
+                                             command->body, command->len) < 0)
+        goto err;
+    } else {
+      log_notice(LD_NET,"Got Extended ORPort command we don't regognize (%u).",
+                 command->cmd);
+    }
+
+    ext_or_cmd_free(command);
+  }
+
+  return 0;
+
+ err:
+  ext_or_cmd_free(command);
+  connection_mark_for_close(conn);
+  return -1;
+}
+
+/** <b>conn</b> finished flushing Extended ORPort messages to the
+ *  network, and is now ready to accept OR traffic. This function
+ *  does the transition. */
+int
+connection_ext_or_finished_flushing(or_connection_t *conn)
+{
+  if (conn->base_.state == EXT_OR_CONN_STATE_FLUSHING) {
+    connection_start_reading(TO_CONN(conn));
+    connection_ext_or_transition(conn);
+  }
+  return 0;
+}
+
+/** Initiate Extended ORPort authentication, by sending the list of
+ *  supported authentication types to the client. */
+int
+connection_ext_or_start_auth(or_connection_t *or_conn)
+{
+  connection_t *conn = TO_CONN(or_conn);
+  const uint8_t authtypes[] = {
+    /* We only support authtype '1' for now. */
+    EXT_OR_AUTHTYPE_SAFECOOKIE,
+    /* Marks the end of the list. */
+    0
+  };
+
+  log_debug(LD_GENERAL,
+           "ExtORPort authentication: Sending supported authentication types");
+
+  connection_buf_add((const char *)authtypes, sizeof(authtypes), conn);
+  conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE;
+
+  return 0;
+}
+
+/** Free any leftover allocated memory of the ext_orport.c subsystem. */
+void
+ext_orport_free_all(void)
+{
+  if (ext_or_auth_cookie) /* Free the auth cookie */
+    tor_free(ext_or_auth_cookie);
+}
+
diff --git a/src/feature/relay/ext_orport.h b/src/feature/relay/ext_orport.h
new file mode 100644
index 0000000000..7eebfdb25b
--- /dev/null
+++ b/src/feature/relay/ext_orport.h
@@ -0,0 +1,64 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef EXT_ORPORT_H
+#define EXT_ORPORT_H
+
+/** States of the Extended ORPort protocol. Be careful before changing
+ *  the numbers: they matter. */
+#define EXT_OR_CONN_STATE_MIN_ 1
+/** Extended ORPort authentication is waiting for the authentication
+ *  type selected by the client. */
+#define EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE 1
+/** Extended ORPort authentication is waiting for the client nonce. */
+#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE 2
+/** Extended ORPort authentication is waiting for the client hash. */
+#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH 3
+#define EXT_OR_CONN_STATE_AUTH_MAX 3
+/** Authentication finished and the Extended ORPort is now accepting
+ *  traffic. */
+#define EXT_OR_CONN_STATE_OPEN 4
+/** Extended ORPort is flushing its last messages and preparing to
+ *  start accepting OR connections. */
+#define EXT_OR_CONN_STATE_FLUSHING 5
+#define EXT_OR_CONN_STATE_MAX_ 5
+
+int connection_ext_or_start_auth(or_connection_t *or_conn);
+
+ext_or_cmd_t *ext_or_cmd_new(uint16_t len);
+
+#define ext_or_cmd_free(cmd)                            \
+  FREE_AND_NULL(ext_or_cmd_t, ext_or_cmd_free_, (cmd))
+
+void ext_or_cmd_free_(ext_or_cmd_t *cmd);
+void connection_or_set_ext_or_identifier(or_connection_t *conn);
+void connection_or_remove_from_ext_or_id_map(or_connection_t *conn);
+void connection_or_clear_ext_or_id_map(void);
+or_connection_t *connection_or_get_by_ext_or_id(const char *id);
+
+int connection_ext_or_finished_flushing(or_connection_t *conn);
+int connection_ext_or_process_inbuf(or_connection_t *or_conn);
+
+int init_ext_or_cookie_authentication(int is_enabled);
+char *get_ext_or_auth_cookie_file_name(void);
+void ext_orport_free_all(void);
+
+#ifdef EXT_ORPORT_PRIVATE
+STATIC int connection_write_ext_or_command(connection_t *conn,
+                                           uint16_t command,
+                                           const char *body,
+                                           size_t bodylen);
+STATIC int handle_client_auth_nonce(const char *client_nonce,
+                         size_t client_nonce_len,
+                         char **client_hash_out,
+                         char **reply_out, size_t *reply_len_out);
+#ifdef TOR_UNIT_TESTS
+extern uint8_t *ext_or_auth_cookie;
+extern int ext_or_auth_cookie_is_set;
+#endif
+#endif /* defined(EXT_ORPORT_PRIVATE) */
+
+#endif /* !defined(EXT_ORPORT_H) */
diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c
new file mode 100644
index 0000000000..44af1e3108
--- /dev/null
+++ b/src/feature/relay/router.c
@@ -0,0 +1,3832 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define ROUTER_PRIVATE
+
+#include "or/or.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "or/directory.h"
+#include "or/dirserv.h"
+#include "or/dns.h"
+#include "or/geoip.h"
+#include "or/hibernate.h"
+#include "or/main.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/protover.h"
+#include "or/relay.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerkeys.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/statefile.h"
+#include "or/torcert.h"
+#include "or/transports.h"
+#include "or/routerset.h"
+
+#include "or/dirauth/mode.h"
+
+#include "or/authority_cert_st.h"
+#include "or/crypt_path_st.h"
+#include "or/dir_connection_st.h"
+#include "or/dir_server_st.h"
+#include "or/extend_info_st.h"
+#include "or/extrainfo_st.h"
+#include "or/node_st.h"
+#include "or/origin_circuit_st.h"
+#include "or/or_state_st.h"
+#include "or/port_cfg_st.h"
+#include "or/routerinfo_st.h"
+
+#include "lib/osinfo/uname.h"
+#include "lib/tls/tortls.h"
+#include "lib/encoding/confline.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+/**
+ * \file router.c
+ * \brief Miscellaneous relay functionality, including RSA key maintenance,
+ * generating and uploading server descriptors, picking an address to
+ * advertise, and so on.
+ *
+ * This module handles the job of deciding whether we are a Tor relay, and if
+ * so what kind. (Mostly through functions like server_mode() that inspect an
+ * or_options_t, but in some cases based on our own capabilities, such as when
+ * we are deciding whether to be a directory cache in
+ * router_has_bandwidth_to_be_dirserver().)
+ *
+ * Also in this module are the functions to generate our own routerinfo_t and
+ * extrainfo_t, and to encode those to signed strings for upload to the
+ * directory authorities.
+ *
+ * This module also handles key maintenance for RSA and Curve25519-ntor keys,
+ * and for our TLS context. (These functions should eventually move to
+ * routerkeys.c along with the code that handles Ed25519 keys now.)
+ **/
+
+/************************************************************/
+
+/*****
+ * Key management: ORs only.
+ *****/
+
+/** Private keys for this OR.  There is also an SSL key managed by tortls.c.
+ */
+static tor_mutex_t *key_lock=NULL;
+static time_t onionkey_set_at=0; /**< When was onionkey last changed? */
+/** Current private onionskin decryption key: used to decode CREATE cells. */
+static crypto_pk_t *onionkey=NULL;
+/** Previous private onionskin decryption key: used to decode CREATE cells
+ * generated by clients that have an older version of our descriptor. */
+static crypto_pk_t *lastonionkey=NULL;
+/** Current private ntor secret key: used to perform the ntor handshake. */
+static curve25519_keypair_t curve25519_onion_key;
+/** Previous private ntor secret key: used to perform the ntor handshake
+ * with clients that have an older version of our descriptor. */
+static curve25519_keypair_t last_curve25519_onion_key;
+/** Private server "identity key": used to sign directory info and TLS
+ * certificates. Never changes. */
+static crypto_pk_t *server_identitykey=NULL;
+/** Digest of server_identitykey. */
+static char server_identitykey_digest[DIGEST_LEN];
+/** Private client "identity key": used to sign bridges' and clients'
+ * outbound TLS certificates. Regenerated on startup and on IP address
+ * change. */
+static crypto_pk_t *client_identitykey=NULL;
+/** Signing key used for v3 directory material; only set for authorities. */
+static crypto_pk_t *authority_signing_key = NULL;
+/** Key certificate to authenticate v3 directory material; only set for
+ * authorities. */
+static authority_cert_t *authority_key_certificate = NULL;
+
+/** For emergency V3 authority key migration: An extra signing key that we use
+ * with our old (obsolete) identity key for a while. */
+static crypto_pk_t *legacy_signing_key = NULL;
+/** For emergency V3 authority key migration: An extra certificate to
+ * authenticate legacy_signing_key with our obsolete identity key.*/
+static authority_cert_t *legacy_key_certificate = NULL;
+
+/* (Note that v3 authorities also have a separate "authority identity key",
+ * but this key is never actually loaded by the Tor process.  Instead, it's
+ * used by tor-gencert to sign new signing keys and make new key
+ * certificates. */
+
+const char *format_node_description(char *buf,
+                                    const char *id_digest,
+                                    int is_named,
+                                    const char *nickname,
+                                    const tor_addr_t *addr,
+                                    uint32_t addr32h);
+
+/** Return a readonly string with human readable description
+ * of <b>err</b>.
+ */
+const char *
+routerinfo_err_to_string(int err)
+{
+  switch (err) {
+    case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR:
+      return "No known exit address yet";
+    case TOR_ROUTERINFO_ERROR_CANNOT_PARSE:
+      return "Cannot parse descriptor";
+    case TOR_ROUTERINFO_ERROR_NOT_A_SERVER:
+      return "Not running in server mode";
+    case TOR_ROUTERINFO_ERROR_DIGEST_FAILED:
+      return "Key digest failed";
+    case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE:
+      return "Cannot generate descriptor";
+    case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
+      return "Descriptor still rebuilding - not ready yet";
+  }
+
+  log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err);
+  tor_assert_unreached();
+
+  return "Unknown error";
+}
+
+/** Return true if we expect given error to be transient.
+ * Return false otherwise.
+ */
+int
+routerinfo_err_is_transient(int err)
+{
+  switch (err) {
+    case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR:
+      return 1;
+    case TOR_ROUTERINFO_ERROR_CANNOT_PARSE:
+      return 1;
+    case TOR_ROUTERINFO_ERROR_NOT_A_SERVER:
+      return 0;
+    case TOR_ROUTERINFO_ERROR_DIGEST_FAILED:
+      return 0; // XXX: bug?
+    case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE:
+      return 1;
+    case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
+      return 1;
+  }
+
+  return 0;
+}
+
+/** Replace the current onion key with <b>k</b>.  Does not affect
+ * lastonionkey; to update lastonionkey correctly, call rotate_onion_key().
+ */
+static void
+set_onion_key(crypto_pk_t *k)
+{
+  if (onionkey && crypto_pk_eq_keys(onionkey, k)) {
+    /* k is already our onion key; free it and return */
+    crypto_pk_free(k);
+    return;
+  }
+  tor_mutex_acquire(key_lock);
+  crypto_pk_free(onionkey);
+  onionkey = k;
+  tor_mutex_release(key_lock);
+  mark_my_descriptor_dirty("set onion key");
+}
+
+/** Return the current onion key.  Requires that the onion key has been
+ * loaded or generated. */
+crypto_pk_t *
+get_onion_key(void)
+{
+  tor_assert(onionkey);
+  return onionkey;
+}
+
+/** Store a full copy of the current onion key into *<b>key</b>, and a full
+ * copy of the most recent onion key into *<b>last</b>.  Store NULL into
+ * a pointer if the corresponding key does not exist.
+ */
+void
+dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
+{
+  tor_assert(key);
+  tor_assert(last);
+  tor_mutex_acquire(key_lock);
+  if (onionkey)
+    *key = crypto_pk_copy_full(onionkey);
+  else
+    *key = NULL;
+  if (lastonionkey)
+    *last = crypto_pk_copy_full(lastonionkey);
+  else
+    *last = NULL;
+  tor_mutex_release(key_lock);
+}
+
+/** Expire our old set of onion keys. This is done by setting
+ * last_curve25519_onion_key and lastonionkey to all zero's and NULL
+ * respectively.
+ *
+ * This function does not perform any grace period checks for the old onion
+ * keys.
+ */
+void
+expire_old_onion_keys(void)
+{
+  char *fname = NULL;
+
+  tor_mutex_acquire(key_lock);
+
+  /* Free lastonionkey and set it to NULL. */
+  if (lastonionkey) {
+    crypto_pk_free(lastonionkey);
+    lastonionkey = NULL;
+  }
+
+  /* We zero out the keypair. See the tor_mem_is_zero() check made in
+   * construct_ntor_key_map() below. */
+  memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
+
+  tor_mutex_release(key_lock);
+
+  fname = get_keydir_fname("secret_onion_key.old");
+  if (file_status(fname) == FN_FILE) {
+    if (tor_unlink(fname) != 0) {
+      log_warn(LD_FS, "Couldn't unlink old onion key file %s: %s",
+               fname, strerror(errno));
+    }
+  }
+  tor_free(fname);
+
+  fname = get_keydir_fname("secret_onion_key_ntor.old");
+  if (file_status(fname) == FN_FILE) {
+    if (tor_unlink(fname) != 0) {
+      log_warn(LD_FS, "Couldn't unlink old ntor onion key file %s: %s",
+               fname, strerror(errno));
+    }
+  }
+  tor_free(fname);
+}
+
+/** Return the current secret onion key for the ntor handshake. Must only
+ * be called from the main thread. */
+static const curve25519_keypair_t *
+get_current_curve25519_keypair(void)
+{
+  return &curve25519_onion_key;
+}
+/** Return a map from KEYID (the key itself) to keypairs for use in the ntor
+ * handshake. Must only be called from the main thread. */
+di_digest256_map_t *
+construct_ntor_key_map(void)
+{
+  di_digest256_map_t *m = NULL;
+
+  if (!tor_mem_is_zero((const char*)
+                       curve25519_onion_key.pubkey.public_key,
+                       CURVE25519_PUBKEY_LEN)) {
+    dimap_add_entry(&m,
+                    curve25519_onion_key.pubkey.public_key,
+                    tor_memdup(&curve25519_onion_key,
+                               sizeof(curve25519_keypair_t)));
+  }
+  if (!tor_mem_is_zero((const char*)
+                          last_curve25519_onion_key.pubkey.public_key,
+                       CURVE25519_PUBKEY_LEN)) {
+    dimap_add_entry(&m,
+                    last_curve25519_onion_key.pubkey.public_key,
+                    tor_memdup(&last_curve25519_onion_key,
+                               sizeof(curve25519_keypair_t)));
+  }
+
+  return m;
+}
+/** Helper used to deallocate a di_digest256_map_t returned by
+ * construct_ntor_key_map. */
+static void
+ntor_key_map_free_helper(void *arg)
+{
+  curve25519_keypair_t *k = arg;
+  memwipe(k, 0, sizeof(*k));
+  tor_free(k);
+}
+/** Release all storage from a keymap returned by construct_ntor_key_map. */
+void
+ntor_key_map_free_(di_digest256_map_t *map)
+{
+  if (!map)
+    return;
+  dimap_free(map, ntor_key_map_free_helper);
+}
+
+/** Return the time when the onion key was last set.  This is either the time
+ * when the process launched, or the time of the most recent key rotation since
+ * the process launched.
+ */
+time_t
+get_onion_key_set_at(void)
+{
+  return onionkey_set_at;
+}
+
+/** Set the current server identity key to <b>k</b>.
+ */
+void
+set_server_identity_key(crypto_pk_t *k)
+{
+  crypto_pk_free(server_identitykey);
+  server_identitykey = k;
+  if (crypto_pk_get_digest(server_identitykey,
+                           server_identitykey_digest) < 0) {
+    log_err(LD_BUG, "Couldn't compute our own identity key digest.");
+    tor_assert(0);
+  }
+}
+
+/** Make sure that we have set up our identity keys to match or not match as
+ * appropriate, and die with an assertion if we have not. */
+static void
+assert_identity_keys_ok(void)
+{
+  if (1)
+    return;
+  tor_assert(client_identitykey);
+  if (public_server_mode(get_options())) {
+    /* assert that we have set the client and server keys to be equal */
+    tor_assert(server_identitykey);
+    tor_assert(crypto_pk_eq_keys(client_identitykey, server_identitykey));
+  } else {
+    /* assert that we have set the client and server keys to be unequal */
+    if (server_identitykey)
+      tor_assert(!crypto_pk_eq_keys(client_identitykey, server_identitykey));
+  }
+}
+
+/** Returns the current server identity key; requires that the key has
+ * been set, and that we are running as a Tor server.
+ */
+crypto_pk_t *
+get_server_identity_key(void)
+{
+  tor_assert(server_identitykey);
+  tor_assert(server_mode(get_options()));
+  assert_identity_keys_ok();
+  return server_identitykey;
+}
+
+/** Return true iff we are a server and the server identity key
+ * has been set. */
+int
+server_identity_key_is_set(void)
+{
+  return server_mode(get_options()) && server_identitykey != NULL;
+}
+
+/** Set the current client identity key to <b>k</b>.
+ */
+void
+set_client_identity_key(crypto_pk_t *k)
+{
+  crypto_pk_free(client_identitykey);
+  client_identitykey = k;
+}
+
+/** Returns the current client identity key for use on outgoing TLS
+ * connections; requires that the key has been set.
+ */
+crypto_pk_t *
+get_tlsclient_identity_key(void)
+{
+  tor_assert(client_identitykey);
+  assert_identity_keys_ok();
+  return client_identitykey;
+}
+
+/** Return true iff the client identity key has been set. */
+int
+client_identity_key_is_set(void)
+{
+  return client_identitykey != NULL;
+}
+
+/** Return the key certificate for this v3 (voting) authority, or NULL
+ * if we have no such certificate. */
+MOCK_IMPL(authority_cert_t *,
+get_my_v3_authority_cert, (void))
+{
+  return authority_key_certificate;
+}
+
+/** Return the v3 signing key for this v3 (voting) authority, or NULL
+ * if we have no such key. */
+crypto_pk_t *
+get_my_v3_authority_signing_key(void)
+{
+  return authority_signing_key;
+}
+
+/** If we're an authority, and we're using a legacy authority identity key for
+ * emergency migration purposes, return the certificate associated with that
+ * key. */
+authority_cert_t *
+get_my_v3_legacy_cert(void)
+{
+  return legacy_key_certificate;
+}
+
+/** If we're an authority, and we're using a legacy authority identity key for
+ * emergency migration purposes, return that key. */
+crypto_pk_t *
+get_my_v3_legacy_signing_key(void)
+{
+  return legacy_signing_key;
+}
+
+/** Replace the previous onion key with the current onion key, and generate
+ * a new previous onion key.  Immediately after calling this function,
+ * the OR should:
+ *   - schedule all previous cpuworkers to shut down _after_ processing
+ *     pending work.  (This will cause fresh cpuworkers to be generated.)
+ *   - generate and upload a fresh routerinfo.
+ */
+void
+rotate_onion_key(void)
+{
+  char *fname, *fname_prev;
+  crypto_pk_t *prkey = NULL;
+  or_state_t *state = get_or_state();
+  curve25519_keypair_t new_curve25519_keypair;
+  time_t now;
+  fname = get_keydir_fname("secret_onion_key");
+  fname_prev = get_keydir_fname("secret_onion_key.old");
+  /* There isn't much point replacing an old key with an empty file */
+  if (file_status(fname) == FN_FILE) {
+    if (replace_file(fname, fname_prev))
+      goto error;
+  }
+  if (!(prkey = crypto_pk_new())) {
+    log_err(LD_GENERAL,"Error constructing rotated onion key");
+    goto error;
+  }
+  if (crypto_pk_generate_key(prkey)) {
+    log_err(LD_BUG,"Error generating onion key");
+    goto error;
+  }
+  if (crypto_pk_write_private_key_to_filename(prkey, fname)) {
+    log_err(LD_FS,"Couldn't write generated onion key to \"%s\".", fname);
+    goto error;
+  }
+  tor_free(fname);
+  tor_free(fname_prev);
+  fname = get_keydir_fname("secret_onion_key_ntor");
+  fname_prev = get_keydir_fname("secret_onion_key_ntor.old");
+  if (curve25519_keypair_generate(&new_curve25519_keypair, 1) < 0)
+    goto error;
+  /* There isn't much point replacing an old key with an empty file */
+  if (file_status(fname) == FN_FILE) {
+    if (replace_file(fname, fname_prev))
+      goto error;
+  }
+  if (curve25519_keypair_write_to_file(&new_curve25519_keypair, fname,
+                                       "onion") < 0) {
+    log_err(LD_FS,"Couldn't write curve25519 onion key to \"%s\".",fname);
+    goto error;
+  }
+  log_info(LD_GENERAL, "Rotating onion key");
+  tor_mutex_acquire(key_lock);
+  crypto_pk_free(lastonionkey);
+  lastonionkey = onionkey;
+  onionkey = prkey;
+  memcpy(&last_curve25519_onion_key, &curve25519_onion_key,
+         sizeof(curve25519_keypair_t));
+  memcpy(&curve25519_onion_key, &new_curve25519_keypair,
+         sizeof(curve25519_keypair_t));
+  now = time(NULL);
+  state->LastRotatedOnionKey = onionkey_set_at = now;
+  tor_mutex_release(key_lock);
+  mark_my_descriptor_dirty("rotated onion key");
+  or_state_mark_dirty(state, get_options()->AvoidDiskWrites ? now+3600 : 0);
+  goto done;
+ error:
+  log_warn(LD_GENERAL, "Couldn't rotate onion key.");
+  if (prkey)
+    crypto_pk_free(prkey);
+ done:
+  memwipe(&new_curve25519_keypair, 0, sizeof(new_curve25519_keypair));
+  tor_free(fname);
+  tor_free(fname_prev);
+}
+
+/** Log greeting message that points to new relay lifecycle document the
+ * first time this function has been called.
+ */
+static void
+log_new_relay_greeting(void)
+{
+  static int already_logged = 0;
+
+  if (already_logged)
+    return;
+
+  tor_log(LOG_NOTICE, LD_GENERAL, "You are running a new relay. "
+         "Thanks for helping the Tor network! If you wish to know "
+         "what will happen in the upcoming weeks regarding its usage, "
+         "have a look at https://blog.torproject.org/blog/lifecycle-of"
+         "-a-new-relay");
+
+  already_logged = 1;
+}
+
+/** Try to read an RSA key from <b>fname</b>.  If <b>fname</b> doesn't exist
+ * and <b>generate</b> is true, create a new RSA key and save it in
+ * <b>fname</b>.  Return the read/created key, or NULL on error.  Log all
+ * errors at level <b>severity</b>. If <b>log_greeting</b> is non-zero and a
+ * new key was created, log_new_relay_greeting() is called.
+ */
+crypto_pk_t *
+init_key_from_file(const char *fname, int generate, int severity,
+                   int log_greeting)
+{
+  crypto_pk_t *prkey = NULL;
+
+  if (!(prkey = crypto_pk_new())) {
+    tor_log(severity, LD_GENERAL,"Error constructing key");
+    goto error;
+  }
+
+  switch (file_status(fname)) {
+    case FN_DIR:
+    case FN_ERROR:
+      tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname);
+      goto error;
+    /* treat empty key files as if the file doesn't exist, and,
+     * if generate is set, replace the empty file in
+     * crypto_pk_write_private_key_to_filename() */
+    case FN_NOENT:
+    case FN_EMPTY:
+      if (generate) {
+        if (!have_lockfile()) {
+          if (try_locking(get_options(), 0)<0) {
+            /* Make sure that --list-fingerprint only creates new keys
+             * if there is no possibility for a deadlock. */
+            tor_log(severity, LD_FS, "Another Tor process has locked \"%s\". "
+                    "Not writing any new keys.", fname);
+            /*XXXX The 'other process' might make a key in a second or two;
+             * maybe we should wait for it. */
+            goto error;
+          }
+        }
+        log_info(LD_GENERAL, "No key found in \"%s\"; generating fresh key.",
+                 fname);
+        if (crypto_pk_generate_key(prkey)) {
+          tor_log(severity, LD_GENERAL,"Error generating onion key");
+          goto error;
+        }
+        if (crypto_pk_check_key(prkey) <= 0) {
+          tor_log(severity, LD_GENERAL,"Generated key seems invalid");
+          goto error;
+        }
+        log_info(LD_GENERAL, "Generated key seems valid");
+        if (log_greeting) {
+            log_new_relay_greeting();
+        }
+        if (crypto_pk_write_private_key_to_filename(prkey, fname)) {
+          tor_log(severity, LD_FS,
+              "Couldn't write generated key to \"%s\".", fname);
+          goto error;
+        }
+      } else {
+        tor_log(severity, LD_GENERAL, "No key found in \"%s\"", fname);
+        goto error;
+      }
+      return prkey;
+    case FN_FILE:
+      if (crypto_pk_read_private_key_from_filename(prkey, fname)) {
+        tor_log(severity, LD_GENERAL,"Error loading private key.");
+        goto error;
+      }
+      return prkey;
+    default:
+      tor_assert(0);
+  }
+
+ error:
+  if (prkey)
+    crypto_pk_free(prkey);
+  return NULL;
+}
+
+/** Load a curve25519 keypair from the file <b>fname</b>, writing it into
+ * <b>keys_out</b>.  If the file isn't found, or is empty, and <b>generate</b>
+ * is true, create a new keypair and write it into the file.  If there are
+ * errors, log them at level <b>severity</b>. Generate files using <b>tag</b>
+ * in their ASCII wrapper. */
+static int
+init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out,
+                                  const char *fname,
+                                  int generate,
+                                  int severity,
+                                  const char *tag)
+{
+  switch (file_status(fname)) {
+    case FN_DIR:
+    case FN_ERROR:
+      tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname);
+      goto error;
+    /* treat empty key files as if the file doesn't exist, and, if generate
+     * is set, replace the empty file in curve25519_keypair_write_to_file() */
+    case FN_NOENT:
+    case FN_EMPTY:
+      if (generate) {
+        if (!have_lockfile()) {
+          if (try_locking(get_options(), 0)<0) {
+            /* Make sure that --list-fingerprint only creates new keys
+             * if there is no possibility for a deadlock. */
+            tor_log(severity, LD_FS, "Another Tor process has locked \"%s\". "
+                    "Not writing any new keys.", fname);
+            /*XXXX The 'other process' might make a key in a second or two;
+             * maybe we should wait for it. */
+            goto error;
+          }
+        }
+        log_info(LD_GENERAL, "No key found in \"%s\"; generating fresh key.",
+                 fname);
+        if (curve25519_keypair_generate(keys_out, 1) < 0)
+          goto error;
+        if (curve25519_keypair_write_to_file(keys_out, fname, tag)<0) {
+          tor_log(severity, LD_FS,
+              "Couldn't write generated key to \"%s\".", fname);
+          memwipe(keys_out, 0, sizeof(*keys_out));
+          goto error;
+        }
+      } else {
+        log_info(LD_GENERAL, "No key found in \"%s\"", fname);
+      }
+      return 0;
+    case FN_FILE:
+      {
+        char *tag_in=NULL;
+        if (curve25519_keypair_read_from_file(keys_out, &tag_in, fname) < 0) {
+          tor_log(severity, LD_GENERAL,"Error loading private key.");
+          tor_free(tag_in);
+          goto error;
+        }
+        if (!tag_in || strcmp(tag_in, tag)) {
+          tor_log(severity, LD_GENERAL,"Unexpected tag %s on private key.",
+              escaped(tag_in));
+          tor_free(tag_in);
+          goto error;
+        }
+        tor_free(tag_in);
+        return 0;
+      }
+    default:
+      tor_assert(0);
+  }
+
+ error:
+  return -1;
+}
+
+/** Try to load the vote-signing private key and certificate for being a v3
+ * directory authority, and make sure they match.  If <b>legacy</b>, load a
+ * legacy key/cert set for emergency key migration; otherwise load the regular
+ * key/cert set.  On success, store them into *<b>key_out</b> and
+ * *<b>cert_out</b> respectively, and return 0.  On failure, return -1. */
+static int
+load_authority_keyset(int legacy, crypto_pk_t **key_out,
+                      authority_cert_t **cert_out)
+{
+  int r = -1;
+  char *fname = NULL, *cert = NULL;
+  const char *eos = NULL;
+  crypto_pk_t *signing_key = NULL;
+  authority_cert_t *parsed = NULL;
+
+  fname = get_keydir_fname(
+                 legacy ? "legacy_signing_key" : "authority_signing_key");
+  signing_key = init_key_from_file(fname, 0, LOG_ERR, 0);
+  if (!signing_key) {
+    log_warn(LD_DIR, "No version 3 directory key found in %s", fname);
+    goto done;
+  }
+  tor_free(fname);
+  fname = get_keydir_fname(
+               legacy ? "legacy_certificate" : "authority_certificate");
+  cert = read_file_to_str(fname, 0, NULL);
+  if (!cert) {
+    log_warn(LD_DIR, "Signing key found, but no certificate found in %s",
+               fname);
+    goto done;
+  }
+  parsed = authority_cert_parse_from_string(cert, &eos);
+  if (!parsed) {
+    log_warn(LD_DIR, "Unable to parse certificate in %s", fname);
+    goto done;
+  }
+  if (!crypto_pk_eq_keys(signing_key, parsed->signing_key)) {
+    log_warn(LD_DIR, "Stored signing key does not match signing key in "
+             "certificate");
+    goto done;
+  }
+
+  crypto_pk_free(*key_out);
+  authority_cert_free(*cert_out);
+
+  *key_out = signing_key;
+  *cert_out = parsed;
+  r = 0;
+  signing_key = NULL;
+  parsed = NULL;
+
+ done:
+  tor_free(fname);
+  tor_free(cert);
+  crypto_pk_free(signing_key);
+  authority_cert_free(parsed);
+  return r;
+}
+
+/** Load the v3 (voting) authority signing key and certificate, if they are
+ * present.  Return -1 if anything is missing, mismatched, or unloadable;
+ * return 0 on success. */
+static int
+init_v3_authority_keys(void)
+{
+  if (load_authority_keyset(0, &authority_signing_key,
+                            &authority_key_certificate)<0)
+    return -1;
+
+  if (get_options()->V3AuthUseLegacyKey &&
+      load_authority_keyset(1, &legacy_signing_key,
+                            &legacy_key_certificate)<0)
+    return -1;
+
+  return 0;
+}
+
+/** If we're a v3 authority, check whether we have a certificate that's
+ * likely to expire soon.  Warn if we do, but not too often. */
+void
+v3_authority_check_key_expiry(void)
+{
+  time_t now, expires;
+  static time_t last_warned = 0;
+  int badness, time_left, warn_interval;
+  if (!authdir_mode_v3(get_options()) || !authority_key_certificate)
+    return;
+
+  now = time(NULL);
+  expires = authority_key_certificate->expires;
+  time_left = (int)( expires - now );
+  if (time_left <= 0) {
+    badness = LOG_ERR;
+    warn_interval = 60*60;
+  } else if (time_left <= 24*60*60) {
+    badness = LOG_WARN;
+    warn_interval = 60*60;
+  } else if (time_left <= 24*60*60*7) {
+    badness = LOG_WARN;
+    warn_interval = 24*60*60;
+  } else if (time_left <= 24*60*60*30) {
+    badness = LOG_WARN;
+    warn_interval = 24*60*60*5;
+  } else {
+    return;
+  }
+
+  if (last_warned + warn_interval > now)
+    return;
+
+  if (time_left <= 0) {
+    tor_log(badness, LD_DIR, "Your v3 authority certificate has expired."
+            " Generate a new one NOW.");
+  } else if (time_left <= 24*60*60) {
+    tor_log(badness, LD_DIR, "Your v3 authority certificate expires in %d "
+            "hours; Generate a new one NOW.", time_left/(60*60));
+  } else {
+    tor_log(badness, LD_DIR, "Your v3 authority certificate expires in %d "
+            "days; Generate a new one soon.", time_left/(24*60*60));
+  }
+  last_warned = now;
+}
+
+/** Get the lifetime of an onion key in days. This value is defined by the
+ * network consesus parameter "onion-key-rotation-days". Always returns a value
+ * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and
+ * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>.
+ */
+static int
+get_onion_key_rotation_days_(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "onion-key-rotation-days",
+                                 DEFAULT_ONION_KEY_LIFETIME_DAYS,
+                                 MIN_ONION_KEY_LIFETIME_DAYS,
+                                 MAX_ONION_KEY_LIFETIME_DAYS);
+}
+
+/** Get the current lifetime of an onion key in seconds. This value is defined
+ * by the network consesus parameter "onion-key-rotation-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_lifetime(void)
+{
+  return get_onion_key_rotation_days_()*24*60*60;
+}
+
+/** Get the grace period of an onion key in seconds. This value is defined by
+ * the network consesus parameter "onion-key-grace-period-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_grace_period(void)
+{
+  int grace_period;
+  grace_period = networkstatus_get_param(NULL,
+                                         "onion-key-grace-period-days",
+                                         DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS,
+                                         MIN_ONION_KEY_GRACE_PERIOD_DAYS,
+                                         get_onion_key_rotation_days_());
+  return grace_period*24*60*60;
+}
+
+/** Set up Tor's TLS contexts, based on our configuration and keys. Return 0
+ * on success, and -1 on failure. */
+int
+router_initialize_tls_context(void)
+{
+  unsigned int flags = 0;
+  const or_options_t *options = get_options();
+  int lifetime = options->SSLKeyLifetime;
+  if (public_server_mode(options))
+    flags |= TOR_TLS_CTX_IS_PUBLIC_SERVER;
+  if (!lifetime) { /* we should guess a good ssl cert lifetime */
+
+    /* choose between 5 and 365 days, and round to the day */
+    unsigned int five_days = 5*24*3600;
+    unsigned int one_year = 365*24*3600;
+    lifetime = crypto_rand_int_range(five_days, one_year);
+    lifetime -= lifetime % (24*3600);
+
+    if (crypto_rand_int(2)) {
+      /* Half the time we expire at midnight, and half the time we expire
+       * one second before midnight. (Some CAs wobble their expiry times a
+       * bit in practice, perhaps to reduce collision attacks; see ticket
+       * 8443 for details about observed certs in the wild.) */
+      lifetime--;
+    }
+  }
+
+  /* It's ok to pass lifetime in as an unsigned int, since
+   * config_parse_interval() checked it. */
+  return tor_tls_context_init(flags,
+                              get_tlsclient_identity_key(),
+                              server_mode(options) ?
+                              get_server_identity_key() : NULL,
+                              (unsigned int)lifetime);
+}
+
+/** Compute fingerprint (or hashed fingerprint if hashed is 1) and write
+ * it to 'fingerprint' (or 'hashed-fingerprint'). Return 0 on success, or
+ * -1 if Tor should die,
+ */
+STATIC int
+router_write_fingerprint(int hashed)
+{
+  char *keydir = NULL, *cp = NULL;
+  const char *fname = hashed ? "hashed-fingerprint" :
+                               "fingerprint";
+  char fingerprint[FINGERPRINT_LEN+1];
+  const or_options_t *options = get_options();
+  char *fingerprint_line = NULL;
+  int result = -1;
+
+  keydir = get_datadir_fname(fname);
+  log_info(LD_GENERAL,"Dumping %sfingerprint to \"%s\"...",
+           hashed ? "hashed " : "", keydir);
+  if (!hashed) {
+    if (crypto_pk_get_fingerprint(get_server_identity_key(),
+                                  fingerprint, 0) < 0) {
+      log_err(LD_GENERAL,"Error computing fingerprint");
+      goto done;
+    }
+  } else {
+    if (crypto_pk_get_hashed_fingerprint(get_server_identity_key(),
+                                         fingerprint) < 0) {
+      log_err(LD_GENERAL,"Error computing hashed fingerprint");
+      goto done;
+    }
+  }
+
+  tor_asprintf(&fingerprint_line, "%s %s\n", options->Nickname, fingerprint);
+
+  /* Check whether we need to write the (hashed-)fingerprint file. */
+
+  cp = read_file_to_str(keydir, RFTS_IGNORE_MISSING, NULL);
+  if (!cp || strcmp(cp, fingerprint_line)) {
+    if (write_str_to_file(keydir, fingerprint_line, 0)) {
+      log_err(LD_FS, "Error writing %sfingerprint line to file",
+              hashed ? "hashed " : "");
+      goto done;
+    }
+  }
+
+  log_notice(LD_GENERAL, "Your Tor %s identity key fingerprint is '%s %s'",
+             hashed ? "bridge's hashed" : "server's", options->Nickname,
+             fingerprint);
+
+  result = 0;
+ done:
+  tor_free(cp);
+  tor_free(keydir);
+  tor_free(fingerprint_line);
+  return result;
+}
+
+static int
+init_keys_common(void)
+{
+  if (!key_lock)
+    key_lock = tor_mutex_new();
+
+  /* There are a couple of paths that put us here before we've asked
+   * openssl to initialize itself. */
+  if (crypto_global_init(get_options()->HardwareAccel,
+                         get_options()->AccelName,
+                         get_options()->AccelDir)) {
+    log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
+    return -1;
+  }
+
+  return 0;
+}
+
+int
+init_keys_client(void)
+{
+  crypto_pk_t *prkey;
+  if (init_keys_common() < 0)
+    return -1;
+
+  if (!(prkey = crypto_pk_new()))
+    return -1;
+  if (crypto_pk_generate_key(prkey)) {
+    crypto_pk_free(prkey);
+    return -1;
+  }
+  set_client_identity_key(prkey);
+  /* Create a TLS context. */
+  if (router_initialize_tls_context() < 0) {
+    log_err(LD_GENERAL,"Error creating TLS context for Tor client.");
+    return -1;
+  }
+  return 0;
+}
+
+/** Initialize all OR private keys, and the TLS context, as necessary.
+ * On OPs, this only initializes the tls context. Return 0 on success,
+ * or -1 if Tor should die.
+ */
+int
+init_keys(void)
+{
+  char *keydir;
+  const char *mydesc;
+  crypto_pk_t *prkey;
+  char digest[DIGEST_LEN];
+  char v3_digest[DIGEST_LEN];
+  const or_options_t *options = get_options();
+  dirinfo_type_t type;
+  time_t now = time(NULL);
+  dir_server_t *ds;
+  int v3_digest_set = 0;
+  authority_cert_t *cert = NULL;
+
+  /* OP's don't need persistent keys; just make up an identity and
+   * initialize the TLS context. */
+  if (!server_mode(options)) {
+    return init_keys_client();
+  }
+  if (init_keys_common() < 0)
+    return -1;
+
+  if (create_keys_directory(options) < 0)
+    return -1;
+
+  /* 1a. Read v3 directory authority key/cert information. */
+  memset(v3_digest, 0, sizeof(v3_digest));
+  if (authdir_mode_v3(options)) {
+    if (init_v3_authority_keys()<0) {
+      log_err(LD_GENERAL, "We're configured as a V3 authority, but we "
+              "were unable to load our v3 authority keys and certificate! "
+              "Use tor-gencert to generate them. Dying.");
+      return -1;
+    }
+    cert = get_my_v3_authority_cert();
+    if (cert) {
+      if (crypto_pk_get_digest(get_my_v3_authority_cert()->identity_key,
+                               v3_digest) < 0) {
+        log_err(LD_BUG, "Couldn't compute my v3 authority identity key "
+                "digest.");
+        return -1;
+      }
+      v3_digest_set = 1;
+    }
+  }
+
+  /* 1b. Read identity key. Make it if none is found. */
+  keydir = get_keydir_fname("secret_id_key");
+  log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir);
+  prkey = init_key_from_file(keydir, 1, LOG_ERR, 1);
+  tor_free(keydir);
+  if (!prkey) return -1;
+  set_server_identity_key(prkey);
+
+  /* 1c. If we are configured as a bridge, generate a client key;
+   * otherwise, set the server identity key as our client identity
+   * key. */
+  if (public_server_mode(options)) {
+    set_client_identity_key(crypto_pk_dup_key(prkey)); /* set above */
+  } else {
+    if (!(prkey = crypto_pk_new()))
+      return -1;
+    if (crypto_pk_generate_key(prkey)) {
+      crypto_pk_free(prkey);
+      return -1;
+    }
+    set_client_identity_key(prkey);
+  }
+
+  /* 1d. Load all ed25519 keys */
+  const int new_signing_key = load_ed_keys(options,now);
+  if (new_signing_key < 0)
+    return -1;
+
+  /* 2. Read onion key.  Make it if none is found. */
+  keydir = get_keydir_fname("secret_onion_key");
+  log_info(LD_GENERAL,"Reading/making onion key \"%s\"...",keydir);
+  prkey = init_key_from_file(keydir, 1, LOG_ERR, 1);
+  tor_free(keydir);
+  if (!prkey) return -1;
+  set_onion_key(prkey);
+  if (options->command == CMD_RUN_TOR) {
+    /* only mess with the state file if we're actually running Tor */
+    or_state_t *state = get_or_state();
+    if (state->LastRotatedOnionKey > 100 && state->LastRotatedOnionKey < now) {
+      /* We allow for some parsing slop, but we don't want to risk accepting
+       * values in the distant future.  If we did, we might never rotate the
+       * onion key. */
+      onionkey_set_at = state->LastRotatedOnionKey;
+    } else {
+      /* We have no LastRotatedOnionKey set; either we just created the key
+       * or it's a holdover from 0.1.2.4-alpha-dev or earlier.  In either case,
+       * start the clock ticking now so that we will eventually rotate it even
+       * if we don't stay up for the full lifetime of an onion key. */
+      state->LastRotatedOnionKey = onionkey_set_at = now;
+      or_state_mark_dirty(state, options->AvoidDiskWrites ?
+                                   time(NULL)+3600 : 0);
+    }
+  }
+
+  keydir = get_keydir_fname("secret_onion_key.old");
+  if (!lastonionkey && file_status(keydir) == FN_FILE) {
+    /* Load keys from non-empty files only.
+     * Missing old keys won't be replaced with freshly generated keys. */
+    prkey = init_key_from_file(keydir, 0, LOG_ERR, 0);
+    if (prkey)
+      lastonionkey = prkey;
+  }
+  tor_free(keydir);
+
+  {
+    /* 2b. Load curve25519 onion keys. */
+    int r;
+    keydir = get_keydir_fname("secret_onion_key_ntor");
+    r = init_curve25519_keypair_from_file(&curve25519_onion_key,
+                                          keydir, 1, LOG_ERR, "onion");
+    tor_free(keydir);
+    if (r<0)
+      return -1;
+
+    keydir = get_keydir_fname("secret_onion_key_ntor.old");
+    if (tor_mem_is_zero((const char *)
+                           last_curve25519_onion_key.pubkey.public_key,
+                        CURVE25519_PUBKEY_LEN) &&
+        file_status(keydir) == FN_FILE) {
+      /* Load keys from non-empty files only.
+       * Missing old keys won't be replaced with freshly generated keys. */
+      init_curve25519_keypair_from_file(&last_curve25519_onion_key,
+                                        keydir, 0, LOG_ERR, "onion");
+    }
+    tor_free(keydir);
+  }
+
+  /* 3. Initialize link key and TLS context. */
+  if (router_initialize_tls_context() < 0) {
+    log_err(LD_GENERAL,"Error initializing TLS context");
+    return -1;
+  }
+
+  /* 3b. Get an ed25519 link certificate.  Note that we need to do this
+   * after we set up the TLS context */
+  if (generate_ed_link_cert(options, now, new_signing_key > 0) < 0) {
+    log_err(LD_GENERAL,"Couldn't make link cert");
+    return -1;
+  }
+
+  /* 4. Build our router descriptor. */
+  /* Must be called after keys are initialized. */
+  mydesc = router_get_my_descriptor();
+  if (authdir_mode_v3(options)) {
+    const char *m = NULL;
+    routerinfo_t *ri;
+    /* We need to add our own fingerprint so it gets recognized. */
+    if (dirserv_add_own_fingerprint(get_server_identity_key())) {
+      log_err(LD_GENERAL,"Error adding own fingerprint to set of relays");
+      return -1;
+    }
+    if (mydesc) {
+      was_router_added_t added;
+      ri = router_parse_entry_from_string(mydesc, NULL, 1, 0, NULL, NULL);
+      if (!ri) {
+        log_err(LD_GENERAL,"Generated a routerinfo we couldn't parse.");
+        return -1;
+      }
+      added = dirserv_add_descriptor(ri, &m, "self");
+      if (!WRA_WAS_ADDED(added)) {
+        if (!WRA_WAS_OUTDATED(added)) {
+          log_err(LD_GENERAL, "Unable to add own descriptor to directory: %s",
+                  m?m:"<unknown error>");
+          return -1;
+        } else {
+          /* If the descriptor was outdated, that's ok. This can happen
+           * when some config options are toggled that affect workers, but
+           * we don't really need new keys yet so the descriptor doesn't
+           * change and the old one is still fresh. */
+          log_info(LD_GENERAL, "Couldn't add own descriptor to directory "
+                   "after key init: %s This is usually not a problem.",
+                   m?m:"<unknown error>");
+        }
+      }
+    }
+  }
+
+  /* 5. Dump fingerprint and possibly hashed fingerprint to files. */
+  if (router_write_fingerprint(0)) {
+    log_err(LD_FS, "Error writing fingerprint to file");
+    return -1;
+  }
+  if (!public_server_mode(options) && router_write_fingerprint(1)) {
+    log_err(LD_FS, "Error writing hashed fingerprint to file");
+    return -1;
+  }
+
+  if (!authdir_mode(options))
+    return 0;
+  /* 6. [authdirserver only] load approved-routers file */
+  if (dirserv_load_fingerprint_file() < 0) {
+    log_err(LD_GENERAL,"Error loading fingerprints");
+    return -1;
+  }
+  /* 6b. [authdirserver only] add own key to approved directories. */
+  crypto_pk_get_digest(get_server_identity_key(), digest);
+  type = ((options->V3AuthoritativeDir ?
+               (V3_DIRINFO|MICRODESC_DIRINFO|EXTRAINFO_DIRINFO) : NO_DIRINFO) |
+          (options->BridgeAuthoritativeDir ? BRIDGE_DIRINFO : NO_DIRINFO));
+
+  ds = router_get_trusteddirserver_by_digest(digest);
+  if (!ds) {
+    ds = trusted_dir_server_new(options->Nickname, NULL,
+                                router_get_advertised_dir_port(options, 0),
+                                router_get_advertised_or_port(options),
+                                NULL,
+                                digest,
+                                v3_digest,
+                                type, 0.0);
+    if (!ds) {
+      log_err(LD_GENERAL,"We want to be a directory authority, but we "
+              "couldn't add ourselves to the authority list. Failing.");
+      return -1;
+    }
+    dir_server_add(ds);
+  }
+  if (ds->type != type) {
+    log_warn(LD_DIR,  "Configured authority type does not match authority "
+             "type in DirAuthority list.  Adjusting. (%d v %d)",
+             type, ds->type);
+    ds->type = type;
+  }
+  if (v3_digest_set && (ds->type & V3_DIRINFO) &&
+      tor_memneq(v3_digest, ds->v3_identity_digest, DIGEST_LEN)) {
+    log_warn(LD_DIR, "V3 identity key does not match identity declared in "
+             "DirAuthority line.  Adjusting.");
+    memcpy(ds->v3_identity_digest, v3_digest, DIGEST_LEN);
+  }
+
+  if (cert) { /* add my own cert to the list of known certs */
+    log_info(LD_DIR, "adding my own v3 cert");
+    if (trusted_dirs_load_certs_from_string(
+                      cert->cache_info.signed_descriptor_body,
+                      TRUSTED_DIRS_CERTS_SRC_SELF, 0,
+                      NULL)<0) {
+      log_warn(LD_DIR, "Unable to parse my own v3 cert! Failing.");
+      return -1;
+    }
+  }
+
+  return 0; /* success */
+}
+
+/* Keep track of whether we should upload our server descriptor,
+ * and what type of server we are.
+ */
+
+/** Whether we can reach our ORPort from the outside. */
+static int can_reach_or_port = 0;
+/** Whether we can reach our DirPort from the outside. */
+static int can_reach_dir_port = 0;
+
+/** Forget what we have learned about our reachability status. */
+void
+router_reset_reachability(void)
+{
+  can_reach_or_port = can_reach_dir_port = 0;
+}
+
+/** Return 1 if we won't do reachability checks, because:
+ *   - AssumeReachable is set, or
+ *   - the network is disabled.
+ * Otherwise, return 0.
+ */
+static int
+router_reachability_checks_disabled(const or_options_t *options)
+{
+  return options->AssumeReachable ||
+         net_is_disabled();
+}
+
+/** Return 0 if we need to do an ORPort reachability check, because:
+ *   - no reachability check has been done yet, or
+ *   - we've initiated reachability checks, but none have succeeded.
+ *  Return 1 if we don't need to do an ORPort reachability check, because:
+ *   - we've seen a successful reachability check, or
+ *   - AssumeReachable is set, or
+ *   - the network is disabled.
+ */
+int
+check_whether_orport_reachable(const or_options_t *options)
+{
+  int reach_checks_disabled = router_reachability_checks_disabled(options);
+  return reach_checks_disabled ||
+         can_reach_or_port;
+}
+
+/** Return 0 if we need to do a DirPort reachability check, because:
+ *   - no reachability check has been done yet, or
+ *   - we've initiated reachability checks, but none have succeeded.
+ *  Return 1 if we don't need to do a DirPort reachability check, because:
+ *   - we've seen a successful reachability check, or
+ *   - there is no DirPort set, or
+ *   - AssumeReachable is set, or
+ *   - the network is disabled.
+ */
+int
+check_whether_dirport_reachable(const or_options_t *options)
+{
+  int reach_checks_disabled = router_reachability_checks_disabled(options) ||
+                              !options->DirPort_set;
+  return reach_checks_disabled ||
+         can_reach_dir_port;
+}
+
+/** The lower threshold of remaining bandwidth required to advertise (or
+ * automatically provide) directory services */
+/* XXX Should this be increased? */
+#define MIN_BW_TO_ADVERTISE_DIRSERVER 51200
+
+/** Return true iff we have enough configured bandwidth to advertise or
+ * automatically provide directory services from cache directory
+ * information. */
+static int
+router_has_bandwidth_to_be_dirserver(const or_options_t *options)
+{
+  if (options->BandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) {
+    return 0;
+  }
+  if (options->RelayBandwidthRate > 0 &&
+      options->RelayBandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) {
+    return 0;
+  }
+  return 1;
+}
+
+/** Helper: Return 1 if we have sufficient resources for serving directory
+ * requests, return 0 otherwise.
+ * dir_port is either 0 or the configured DirPort number.
+ * If AccountingMax is set less than our advertised bandwidth, then don't
+ * serve requests. Likewise, if our advertised bandwidth is less than
+ * MIN_BW_TO_ADVERTISE_DIRSERVER, don't bother trying to serve requests.
+ */
+static int
+router_should_be_dirserver(const or_options_t *options, int dir_port)
+{
+  static int advertising=1; /* start out assuming we will advertise */
+  int new_choice=1;
+  const char *reason = NULL;
+
+  if (accounting_is_enabled(options) &&
+    get_options()->AccountingRule != ACCT_IN) {
+    /* Don't spend bytes for directory traffic if we could end up hibernating,
+     * but allow DirPort otherwise. Some relay operators set AccountingMax
+     * because they're confused or to get statistics. Directory traffic has a
+     * much larger effect on output than input so there is no reason to turn it
+     * off if using AccountingRule in. */
+    int interval_length = accounting_get_interval_length();
+    uint32_t effective_bw = get_effective_bwrate(options);
+    uint64_t acc_bytes;
+    if (!interval_length) {
+      log_warn(LD_BUG, "An accounting interval is not allowed to be zero "
+                       "seconds long. Raising to 1.");
+      interval_length = 1;
+    }
+    log_info(LD_GENERAL, "Calculating whether to advertise %s: effective "
+                         "bwrate: %u, AccountingMax: %"PRIu64", "
+                         "accounting interval length %d",
+                         dir_port ? "dirport" : "begindir",
+                         effective_bw, (options->AccountingMax),
+                         interval_length);
+
+    acc_bytes = options->AccountingMax;
+    if (get_options()->AccountingRule == ACCT_SUM)
+      acc_bytes /= 2;
+    if (effective_bw >=
+        acc_bytes / interval_length) {
+      new_choice = 0;
+      reason = "AccountingMax enabled";
+    }
+  } else if (! router_has_bandwidth_to_be_dirserver(options)) {
+    /* if we're advertising a small amount */
+    new_choice = 0;
+    reason = "BandwidthRate under 50KB";
+  }
+
+  if (advertising != new_choice) {
+    if (new_choice == 1) {
+      if (dir_port > 0)
+        log_notice(LD_DIR, "Advertising DirPort as %d", dir_port);
+      else
+        log_notice(LD_DIR, "Advertising directory service support");
+    } else {
+      tor_assert(reason);
+      log_notice(LD_DIR, "Not advertising Dir%s (Reason: %s)",
+                 dir_port ? "Port" : "ectory Service support", reason);
+    }
+    advertising = new_choice;
+  }
+
+  return advertising;
+}
+
+/** Return 1 if we are configured to accept either relay or directory requests
+ * from clients and we aren't at risk of exceeding our bandwidth limits, thus
+ * we should be a directory server. If not, return 0.
+ */
+int
+dir_server_mode(const or_options_t *options)
+{
+  if (!options->DirCache)
+    return 0;
+  return options->DirPort_set ||
+    (server_mode(options) && router_has_bandwidth_to_be_dirserver(options));
+}
+
+/** Look at a variety of factors, and return 0 if we don't want to
+ * advertise the fact that we have a DirPort open or begindir support, else
+ * return 1.
+ *
+ * Where dir_port or supports_tunnelled_dir_requests are not relevant, they
+ * must be 0.
+ *
+ * Log a helpful message if we change our mind about whether to publish.
+ */
+static int
+decide_to_advertise_dir_impl(const or_options_t *options,
+                             uint16_t dir_port,
+                             int supports_tunnelled_dir_requests)
+{
+  /* Part one: reasons to publish or not publish that aren't
+   * worth mentioning to the user, either because they're obvious
+   * or because they're normal behavior. */
+
+  /* short circuit the rest of the function */
+  if (!dir_port && !supports_tunnelled_dir_requests)
+    return 0;
+  if (authdir_mode(options)) /* always publish */
+    return 1;
+  if (net_is_disabled())
+    return 0;
+  if (dir_port && !router_get_advertised_dir_port(options, dir_port))
+    return 0;
+  if (supports_tunnelled_dir_requests &&
+      !router_get_advertised_or_port(options))
+    return 0;
+
+  /* Part two: consider config options that could make us choose to
+   * publish or not publish that the user might find surprising. */
+  return router_should_be_dirserver(options, dir_port);
+}
+
+/** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to
+ * advertise the fact that we have a DirPort open, else return the
+ * DirPort we want to advertise.
+ */
+static int
+router_should_advertise_dirport(const or_options_t *options, uint16_t dir_port)
+{
+  /* supports_tunnelled_dir_requests is not relevant, pass 0 */
+  return decide_to_advertise_dir_impl(options, dir_port, 0) ? dir_port : 0;
+}
+
+/** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to
+ * advertise the fact that we support begindir requests, else return 1.
+ */
+static int
+router_should_advertise_begindir(const or_options_t *options,
+                             int supports_tunnelled_dir_requests)
+{
+  /* dir_port is not relevant, pass 0 */
+  return decide_to_advertise_dir_impl(options, 0,
+                                      supports_tunnelled_dir_requests);
+}
+
+/** Allocate and return a new extend_info_t that can be used to build
+ * a circuit to or through the router <b>r</b>. Uses the primary
+ * address of the router, so should only be called on a server. */
+static extend_info_t *
+extend_info_from_router(const routerinfo_t *r)
+{
+  tor_addr_port_t ap;
+  tor_assert(r);
+
+  /* Make sure we don't need to check address reachability */
+  tor_assert_nonfatal(router_skip_or_reachability(get_options(), 0));
+
+  const ed25519_public_key_t *ed_id_key;
+  if (r->cache_info.signing_key_cert)
+    ed_id_key = &r->cache_info.signing_key_cert->signing_key;
+  else
+    ed_id_key = NULL;
+
+  router_get_prim_orport(r, &ap);
+  return extend_info_new(r->nickname, r->cache_info.identity_digest,
+                         ed_id_key,
+                         r->onion_pkey, r->onion_curve25519_pkey,
+                         &ap.addr, ap.port);
+}
+
+/**See if we currently believe our ORPort or DirPort to be
+ * unreachable. If so, return 1 else return 0.
+ */
+static int
+router_should_check_reachability(int test_or, int test_dir)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const or_options_t *options = get_options();
+
+  if (!me)
+    return 0;
+
+  if (routerset_contains_router(options->ExcludeNodes, me, -1) &&
+      options->StrictNodes) {
+    /* If we've excluded ourself, and StrictNodes is set, we can't test
+     * ourself. */
+    if (test_or || test_dir) {
+#define SELF_EXCLUDED_WARN_INTERVAL 3600
+      static ratelim_t warning_limit=RATELIM_INIT(SELF_EXCLUDED_WARN_INTERVAL);
+      log_fn_ratelim(&warning_limit, LOG_WARN, LD_CIRC,
+                 "Can't peform self-tests for this relay: we have "
+                 "listed ourself in ExcludeNodes, and StrictNodes is set. "
+                 "We cannot learn whether we are usable, and will not "
+                 "be able to advertise ourself.");
+    }
+    return 0;
+  }
+  return 1;
+}
+
+/** Some time has passed, or we just got new directory information.
+ * See if we currently believe our ORPort or DirPort to be
+ * unreachable. If so, launch a new test for it.
+ *
+ * For ORPort, we simply try making a circuit that ends at ourselves.
+ * Success is noticed in onionskin_answer().
+ *
+ * For DirPort, we make a connection via Tor to our DirPort and ask
+ * for our own server descriptor.
+ * Success is noticed in connection_dir_client_reached_eof().
+ */
+void
+router_do_reachability_checks(int test_or, int test_dir)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const or_options_t *options = get_options();
+  int orport_reachable = check_whether_orport_reachable(options);
+  tor_addr_t addr;
+
+  if (router_should_check_reachability(test_or, test_dir)) {
+    if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) {
+      extend_info_t *ei = extend_info_from_router(me);
+      /* XXX IPv6 self testing */
+      log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.",
+               !orport_reachable ? "reachability" : "bandwidth",
+               fmt_addr32(me->addr), me->or_port);
+      circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei,
+                              CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
+      extend_info_free(ei);
+    }
+
+    /* XXX IPv6 self testing */
+    tor_addr_from_ipv4h(&addr, me->addr);
+    if (test_dir && !check_whether_dirport_reachable(options) &&
+        !connection_get_by_type_addr_port_purpose(
+                  CONN_TYPE_DIR, &addr, me->dir_port,
+                  DIR_PURPOSE_FETCH_SERVERDESC)) {
+      tor_addr_port_t my_orport, my_dirport;
+      memcpy(&my_orport.addr, &addr, sizeof(addr));
+      memcpy(&my_dirport.addr, &addr, sizeof(addr));
+      my_orport.port = me->or_port;
+      my_dirport.port = me->dir_port;
+      /* ask myself, via tor, for my server descriptor. */
+      directory_request_t *req =
+        directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC);
+      directory_request_set_or_addr_port(req, &my_orport);
+      directory_request_set_dir_addr_port(req, &my_dirport);
+      directory_request_set_directory_id_digest(req,
+                                              me->cache_info.identity_digest);
+      // ask via an anon circuit, connecting to our dirport.
+      directory_request_set_indirection(req, DIRIND_ANON_DIRPORT);
+      directory_request_set_resource(req, "authority.z");
+      directory_initiate_request(req);
+      directory_request_free(req);
+    }
+  }
+}
+
+/** Annotate that we found our ORPort reachable. */
+void
+router_orport_found_reachable(void)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const or_options_t *options = get_options();
+  if (!can_reach_or_port && me) {
+    char *address = tor_dup_ip(me->addr);
+    log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from "
+               "the outside. Excellent.%s",
+               options->PublishServerDescriptor_ != NO_DIRINFO
+               && check_whether_dirport_reachable(options) ?
+                 " Publishing server descriptor." : "");
+    can_reach_or_port = 1;
+    mark_my_descriptor_dirty("ORPort found reachable");
+    /* This is a significant enough change to upload immediately,
+     * at least in a test network */
+    if (options->TestingTorNetwork == 1) {
+      reschedule_descriptor_update_check();
+    }
+    control_event_server_status(LOG_NOTICE,
+                                "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d",
+                                address, me->or_port);
+    tor_free(address);
+  }
+}
+
+/** Annotate that we found our DirPort reachable. */
+void
+router_dirport_found_reachable(void)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const or_options_t *options = get_options();
+  if (!can_reach_dir_port && me) {
+    char *address = tor_dup_ip(me->addr);
+    log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable "
+               "from the outside. Excellent.%s",
+               options->PublishServerDescriptor_ != NO_DIRINFO
+               && check_whether_orport_reachable(options) ?
+               " Publishing server descriptor." : "");
+    can_reach_dir_port = 1;
+    if (router_should_advertise_dirport(options, me->dir_port)) {
+      mark_my_descriptor_dirty("DirPort found reachable");
+      /* This is a significant enough change to upload immediately,
+       * at least in a test network */
+      if (options->TestingTorNetwork == 1) {
+        reschedule_descriptor_update_check();
+      }
+    }
+    control_event_server_status(LOG_NOTICE,
+                                "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d",
+                                address, me->dir_port);
+    tor_free(address);
+  }
+}
+
+/** We have enough testing circuits open. Send a bunch of "drop"
+ * cells down each of them, to exercise our bandwidth. */
+void
+router_perform_bandwidth_test(int num_circs, time_t now)
+{
+  int num_cells = (int)(get_options()->BandwidthRate * 10 /
+                        CELL_MAX_NETWORK_SIZE);
+  int max_cells = num_cells < CIRCWINDOW_START ?
+                    num_cells : CIRCWINDOW_START;
+  int cells_per_circuit = max_cells / num_circs;
+  origin_circuit_t *circ = NULL;
+
+  log_notice(LD_OR,"Performing bandwidth self-test...done.");
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ, NULL,
+                                              CIRCUIT_PURPOSE_TESTING))) {
+    /* dump cells_per_circuit drop cells onto this circ */
+    int i = cells_per_circuit;
+    if (circ->base_.state != CIRCUIT_STATE_OPEN)
+      continue;
+    circ->base_.timestamp_dirty = now;
+    while (i-- > 0) {
+      if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+                                       RELAY_COMMAND_DROP,
+                                       NULL, 0, circ->cpath->prev)<0) {
+        return; /* stop if error */
+      }
+    }
+  }
+}
+
+/** Return true iff our network is in some sense disabled or shutting down:
+ * either we're hibernating, entering hibernation, or the network is turned
+ * off with DisableNetwork. */
+int
+net_is_disabled(void)
+{
+  return get_options()->DisableNetwork || we_are_hibernating();
+}
+
+/** Return true iff our network is in some sense "completely disabled" either
+ * we're fully hibernating or the network is turned off with
+ * DisableNetwork. */
+int
+net_is_completely_disabled(void)
+{
+  return get_options()->DisableNetwork || we_are_fully_hibernating();
+}
+
+/** Return true iff we believe ourselves to be an authoritative
+ * directory server.
+ */
+int
+authdir_mode(const or_options_t *options)
+{
+  return options->AuthoritativeDir != 0;
+}
+/** Return true iff we are an authoritative directory server that is
+ * authoritative about receiving and serving descriptors of type
+ * <b>purpose</b> on its dirport.
+ */
+int
+authdir_mode_handles_descs(const or_options_t *options, int purpose)
+{
+  if (BUG(purpose < 0)) /* Deprecated. */
+    return authdir_mode(options);
+  else if (purpose == ROUTER_PURPOSE_GENERAL)
+    return authdir_mode_v3(options);
+  else if (purpose == ROUTER_PURPOSE_BRIDGE)
+    return authdir_mode_bridge(options);
+  else
+    return 0;
+}
+/** Return true iff we are an authoritative directory server that
+ * publishes its own network statuses.
+ */
+int
+authdir_mode_publishes_statuses(const or_options_t *options)
+{
+  if (authdir_mode_bridge(options))
+    return 0;
+  return authdir_mode(options);
+}
+/** Return true iff we are an authoritative directory server that
+ * tests reachability of the descriptors it learns about.
+ */
+int
+authdir_mode_tests_reachability(const or_options_t *options)
+{
+  return authdir_mode(options);
+}
+/** Return true iff we believe ourselves to be a bridge authoritative
+ * directory server.
+ */
+int
+authdir_mode_bridge(const or_options_t *options)
+{
+  return authdir_mode(options) && options->BridgeAuthoritativeDir != 0;
+}
+
+/** Return true iff we are trying to be a server.
+ */
+MOCK_IMPL(int,
+server_mode,(const or_options_t *options))
+{
+  if (options->ClientOnly) return 0;
+  return (options->ORPort_set);
+}
+
+/** Return true iff we are trying to be a non-bridge server.
+ */
+MOCK_IMPL(int,
+public_server_mode,(const or_options_t *options))
+{
+  if (!server_mode(options)) return 0;
+  return (!options->BridgeRelay);
+}
+
+/** Return true iff the combination of options in <b>options</b> and parameters
+ * in the consensus mean that we don't want to allow exits from circuits
+ * we got from addresses not known to be servers. */
+int
+should_refuse_unknown_exits(const or_options_t *options)
+{
+  if (options->RefuseUnknownExits != -1) {
+    return options->RefuseUnknownExits;
+  } else {
+    return networkstatus_get_param(NULL, "refuseunknownexits", 1, 0, 1);
+  }
+}
+
+/** Remember if we've advertised ourselves to the dirservers. */
+static int server_is_advertised=0;
+
+/** Return true iff we have published our descriptor lately.
+ */
+MOCK_IMPL(int,
+advertised_server_mode,(void))
+{
+  return server_is_advertised;
+}
+
+/**
+ * Called with a boolean: set whether we have recently published our
+ * descriptor.
+ */
+static void
+set_server_advertised(int s)
+{
+  server_is_advertised = s;
+}
+
+/** Return true iff we are trying to proxy client connections. */
+int
+proxy_mode(const or_options_t *options)
+{
+  (void)options;
+  SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
+    if (p->type == CONN_TYPE_AP_LISTENER ||
+        p->type == CONN_TYPE_AP_TRANS_LISTENER ||
+        p->type == CONN_TYPE_AP_DNS_LISTENER ||
+        p->type == CONN_TYPE_AP_NATD_LISTENER)
+      return 1;
+  } SMARTLIST_FOREACH_END(p);
+  return 0;
+}
+
+/** Decide if we're a publishable server. We are a publishable server if:
+ * - We don't have the ClientOnly option set
+ * and
+ * - We have the PublishServerDescriptor option set to non-empty
+ * and
+ * - We have ORPort set
+ * and
+ * - We believe our ORPort and DirPort (if present) are reachable from
+ *   the outside; or
+ * - We believe our ORPort is reachable from the outside, and we can't
+ *   check our DirPort because the consensus has no exits; or
+ * - We are an authoritative directory server.
+ */
+static int
+decide_if_publishable_server(void)
+{
+  const or_options_t *options = get_options();
+
+  if (options->ClientOnly)
+    return 0;
+  if (options->PublishServerDescriptor_ == NO_DIRINFO)
+    return 0;
+  if (!server_mode(options))
+    return 0;
+  if (authdir_mode(options))
+    return 1;
+  if (!router_get_advertised_or_port(options))
+    return 0;
+  if (!check_whether_orport_reachable(options))
+    return 0;
+  if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL) {
+    /* All set: there are no exits in the consensus (maybe this is a tiny
+     * test network), so we can't check our DirPort reachability. */
+    return 1;
+  } else {
+    return check_whether_dirport_reachable(options);
+  }
+}
+
+/** Initiate server descriptor upload as reasonable (if server is publishable,
+ * etc).  <b>force</b> is as for router_upload_dir_desc_to_dirservers.
+ *
+ * We need to rebuild the descriptor if it's dirty even if we're not
+ * uploading, because our reachability testing *uses* our descriptor to
+ * determine what IP address and ports to test.
+ */
+void
+consider_publishable_server(int force)
+{
+  int rebuilt;
+
+  if (!server_mode(get_options()))
+    return;
+
+  rebuilt = router_rebuild_descriptor(0);
+  if (decide_if_publishable_server()) {
+    set_server_advertised(1);
+    if (rebuilt == 0)
+      router_upload_dir_desc_to_dirservers(force);
+  } else {
+    set_server_advertised(0);
+  }
+}
+
+/** Return the port of the first active listener of type
+ *  <b>listener_type</b>. */
+/** XXX not a very good interface. it's not reliable when there are
+    multiple listeners. */
+uint16_t
+router_get_active_listener_port_by_type_af(int listener_type,
+                                           sa_family_t family)
+{
+  /* Iterate all connections, find one of the right kind and return
+     the port. Not very sophisticated or fast, but effective. */
+  smartlist_t *conns = get_connection_array();
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    if (conn->type == listener_type && !conn->marked_for_close &&
+        conn->socket_family == family) {
+      return conn->port;
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  return 0;
+}
+
+/** Return the port that we should advertise as our ORPort; this is either
+ * the one configured in the ORPort option, or the one we actually bound to
+ * if ORPort is "auto".
+ */
+uint16_t
+router_get_advertised_or_port(const or_options_t *options)
+{
+  return router_get_advertised_or_port_by_af(options, AF_INET);
+}
+
+/** As router_get_advertised_or_port(), but allows an address family argument.
+ */
+uint16_t
+router_get_advertised_or_port_by_af(const or_options_t *options,
+                                    sa_family_t family)
+{
+  int port = get_first_advertised_port_by_type_af(CONN_TYPE_OR_LISTENER,
+                                                  family);
+  (void)options;
+
+  /* If the port is in 'auto' mode, we have to use
+     router_get_listener_port_by_type(). */
+  if (port == CFG_AUTO_PORT)
+    return router_get_active_listener_port_by_type_af(CONN_TYPE_OR_LISTENER,
+                                                      family);
+
+  return port;
+}
+
+/** Return the port that we should advertise as our DirPort;
+ * this is one of three possibilities:
+ * The one that is passed as <b>dirport</b> if the DirPort option is 0, or
+ * the one configured in the DirPort option,
+ * or the one we actually bound to if DirPort is "auto". */
+uint16_t
+router_get_advertised_dir_port(const or_options_t *options, uint16_t dirport)
+{
+  int dirport_configured = get_primary_dir_port();
+  (void)options;
+
+  if (!dirport_configured)
+    return dirport;
+
+  if (dirport_configured == CFG_AUTO_PORT)
+    return router_get_active_listener_port_by_type_af(CONN_TYPE_DIR_LISTENER,
+                                                      AF_INET);
+
+  return dirport_configured;
+}
+
+/*
+ * OR descriptor generation.
+ */
+
+/** My routerinfo. */
+static routerinfo_t *desc_routerinfo = NULL;
+/** My extrainfo */
+static extrainfo_t *desc_extrainfo = NULL;
+/** Why did we most recently decide to regenerate our descriptor?  Used to
+ * tell the authorities why we're sending it to them. */
+static const char *desc_gen_reason = "uninitialized reason";
+/** Since when has our descriptor been "clean"?  0 if we need to regenerate it
+ * now. */
+static time_t desc_clean_since = 0;
+/** Why did we mark the descriptor dirty? */
+static const char *desc_dirty_reason = "Tor just started";
+/** Boolean: do we need to regenerate the above? */
+static int desc_needs_upload = 0;
+
+/** OR only: If <b>force</b> is true, or we haven't uploaded this
+ * descriptor successfully yet, try to upload our signed descriptor to
+ * all the directory servers we know about.
+ */
+void
+router_upload_dir_desc_to_dirservers(int force)
+{
+  const routerinfo_t *ri;
+  extrainfo_t *ei;
+  char *msg;
+  size_t desc_len, extra_len = 0, total_len;
+  dirinfo_type_t auth = get_options()->PublishServerDescriptor_;
+
+  ri = router_get_my_routerinfo();
+  if (!ri) {
+    log_info(LD_GENERAL, "No descriptor; skipping upload");
+    return;
+  }
+  ei = router_get_my_extrainfo();
+  if (auth == NO_DIRINFO)
+    return;
+  if (!force && !desc_needs_upload)
+    return;
+
+  log_info(LD_OR, "Uploading relay descriptor to directory authorities%s",
+           force ? " (forced)" : "");
+
+  desc_needs_upload = 0;
+
+  desc_len = ri->cache_info.signed_descriptor_len;
+  extra_len = ei ? ei->cache_info.signed_descriptor_len : 0;
+  total_len = desc_len + extra_len + 1;
+  msg = tor_malloc(total_len);
+  memcpy(msg, ri->cache_info.signed_descriptor_body, desc_len);
+  if (ei) {
+    memcpy(msg+desc_len, ei->cache_info.signed_descriptor_body, extra_len);
+  }
+  msg[desc_len+extra_len] = 0;
+
+  directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_DIR,
+                               (auth & BRIDGE_DIRINFO) ?
+                                 ROUTER_PURPOSE_BRIDGE :
+                                 ROUTER_PURPOSE_GENERAL,
+                               auth, msg, desc_len, extra_len);
+  tor_free(msg);
+}
+
+/** OR only: Check whether my exit policy says to allow connection to
+ * conn.  Return 0 if we accept; non-0 if we reject.
+ */
+int
+router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  if (!me) /* make sure routerinfo exists */
+    return -1;
+
+  /* make sure it's resolved to something. this way we can't get a
+     'maybe' below. */
+  if (tor_addr_is_null(addr))
+    return -1;
+
+  /* look at router_get_my_routerinfo()->exit_policy for both the v4 and the
+   * v6 policies.  The exit_policy field in router_get_my_routerinfo() is a
+   * bit unusual, in that it contains IPv6 and IPv6 entries.  We don't want to
+   * look at router_get_my_routerinfo()->ipv6_exit_policy, since that's a port
+   * summary. */
+  if ((tor_addr_family(addr) == AF_INET ||
+       tor_addr_family(addr) == AF_INET6)) {
+    return compare_tor_addr_to_addr_policy(addr, port,
+                               me->exit_policy) != ADDR_POLICY_ACCEPTED;
+#if 0
+  } else if (tor_addr_family(addr) == AF_INET6) {
+    return get_options()->IPv6Exit &&
+      desc_routerinfo->ipv6_exit_policy &&
+      compare_tor_addr_to_short_policy(addr, port,
+                               me->ipv6_exit_policy) != ADDR_POLICY_ACCEPTED;
+#endif /* 0 */
+  } else {
+    return -1;
+  }
+}
+
+/** Return true iff my exit policy is reject *:*.  Return -1 if we don't
+ * have a descriptor */
+MOCK_IMPL(int,
+router_my_exit_policy_is_reject_star,(void))
+{
+  if (!router_get_my_routerinfo()) /* make sure routerinfo exists */
+    return -1;
+
+  return router_get_my_routerinfo()->policy_is_reject_star;
+}
+
+/** Return true iff I'm a server and <b>digest</b> is equal to
+ * my server identity key digest. */
+int
+router_digest_is_me(const char *digest)
+{
+  return (server_identitykey &&
+          tor_memeq(server_identitykey_digest, digest, DIGEST_LEN));
+}
+
+/** Return my identity digest. */
+const uint8_t *
+router_get_my_id_digest(void)
+{
+  return (const uint8_t *)server_identitykey_digest;
+}
+
+/** Return true iff I'm a server and <b>digest</b> is equal to
+ * my identity digest. */
+int
+router_extrainfo_digest_is_me(const char *digest)
+{
+  extrainfo_t *ei = router_get_my_extrainfo();
+  if (!ei)
+    return 0;
+
+  return tor_memeq(digest,
+                 ei->cache_info.signed_descriptor_digest,
+                 DIGEST_LEN);
+}
+
+/** A wrapper around router_digest_is_me(). */
+int
+router_is_me(const routerinfo_t *router)
+{
+  return router_digest_is_me(router->cache_info.identity_digest);
+}
+
+/** Return a routerinfo for this OR, rebuilding a fresh one if
+ * necessary.  Return NULL on error, or if called on an OP. */
+MOCK_IMPL(const routerinfo_t *,
+router_get_my_routerinfo,(void))
+{
+  return router_get_my_routerinfo_with_err(NULL);
+}
+
+/** Return routerinfo of this OR. Rebuild it from
+ * scratch if needed. Set <b>*err</b> to 0 on success or to
+ * appropriate TOR_ROUTERINFO_ERROR_* value on failure.
+ */
+MOCK_IMPL(const routerinfo_t *,
+router_get_my_routerinfo_with_err,(int *err))
+{
+  if (!server_mode(get_options())) {
+    if (err)
+      *err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER;
+
+    return NULL;
+  }
+
+  if (!desc_clean_since) {
+    int rebuild_err = router_rebuild_descriptor(0);
+    if (rebuild_err < 0) {
+      if (err)
+        *err = rebuild_err;
+
+      return NULL;
+    }
+  }
+
+  if (!desc_routerinfo) {
+    if (err)
+      *err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
+
+    return NULL;
+  }
+
+  if (err)
+    *err = 0;
+
+  return desc_routerinfo;
+}
+
+/** OR only: Return a signed server descriptor for this OR, rebuilding a fresh
+ * one if necessary.  Return NULL on error.
+ */
+const char *
+router_get_my_descriptor(void)
+{
+  const char *body;
+  const routerinfo_t *me = router_get_my_routerinfo();
+  if (! me)
+    return NULL;
+  tor_assert(me->cache_info.saved_location == SAVED_NOWHERE);
+  body = signed_descriptor_get_body(&me->cache_info);
+  /* Make sure this is nul-terminated. */
+  tor_assert(!body[me->cache_info.signed_descriptor_len]);
+  log_debug(LD_GENERAL,"my desc is '%s'", body);
+  return body;
+}
+
+/** Return the extrainfo document for this OR, or NULL if we have none.
+ * Rebuilt it (and the server descriptor) if necessary. */
+extrainfo_t *
+router_get_my_extrainfo(void)
+{
+  if (!server_mode(get_options()))
+    return NULL;
+  if (router_rebuild_descriptor(0))
+    return NULL;
+  return desc_extrainfo;
+}
+
+/** Return a human-readable string describing what triggered us to generate
+ * our current descriptor, or NULL if we don't know. */
+const char *
+router_get_descriptor_gen_reason(void)
+{
+  return desc_gen_reason;
+}
+
+/** A list of nicknames that we've warned about including in our family
+ * declaration verbatim rather than as digests. */
+static smartlist_t *warned_nonexistent_family = NULL;
+
+static int router_guess_address_from_dir_headers(uint32_t *guess);
+
+/** Make a current best guess at our address, either because
+ * it's configured in torrc, or because we've learned it from
+ * dirserver headers. Place the answer in *<b>addr</b> and return
+ * 0 on success, else return -1 if we have no guess.
+ *
+ * If <b>cache_only</b> is true, just return any cached answers, and
+ * don't try to get any new answers.
+ */
+MOCK_IMPL(int,
+router_pick_published_address,(const or_options_t *options, uint32_t *addr,
+                               int cache_only))
+{
+  /* First, check the cached output from resolve_my_address(). */
+  *addr = get_last_resolved_addr();
+  if (*addr)
+    return 0;
+
+  /* Second, consider doing a resolve attempt right here. */
+  if (!cache_only) {
+    if (resolve_my_address(LOG_INFO, options, addr, NULL, NULL) >= 0) {
+      log_info(LD_CONFIG,"Success: chose address '%s'.", fmt_addr32(*addr));
+      return 0;
+    }
+  }
+
+  /* Third, check the cached output from router_new_address_suggestion(). */
+  if (router_guess_address_from_dir_headers(addr) >= 0)
+    return 0;
+
+  /* We have no useful cached answers. Return failure. */
+  return -1;
+}
+
+/* Like router_check_descriptor_address_consistency, but specifically for the
+ * ORPort or DirPort.
+ * listener_type is either CONN_TYPE_OR_LISTENER or CONN_TYPE_DIR_LISTENER. */
+static void
+router_check_descriptor_address_port_consistency(uint32_t ipv4h_desc_addr,
+                                                 int listener_type)
+{
+  tor_assert(listener_type == CONN_TYPE_OR_LISTENER ||
+             listener_type == CONN_TYPE_DIR_LISTENER);
+
+  /* The first advertised Port may be the magic constant CFG_AUTO_PORT.
+   */
+  int port_v4_cfg = get_first_advertised_port_by_type_af(listener_type,
+                                                         AF_INET);
+  if (port_v4_cfg != 0 &&
+      !port_exists_by_type_addr32h_port(listener_type,
+                                        ipv4h_desc_addr, port_v4_cfg, 1)) {
+        const tor_addr_t *port_addr = get_first_advertised_addr_by_type_af(
+                                                                listener_type,
+                                                                AF_INET);
+        /* If we're building a descriptor with no advertised address,
+         * something is terribly wrong. */
+        tor_assert(port_addr);
+
+        tor_addr_t desc_addr;
+        char port_addr_str[TOR_ADDR_BUF_LEN];
+        char desc_addr_str[TOR_ADDR_BUF_LEN];
+
+        tor_addr_to_str(port_addr_str, port_addr, TOR_ADDR_BUF_LEN, 0);
+
+        tor_addr_from_ipv4h(&desc_addr, ipv4h_desc_addr);
+        tor_addr_to_str(desc_addr_str, &desc_addr, TOR_ADDR_BUF_LEN, 0);
+
+        const char *listener_str = (listener_type == CONN_TYPE_OR_LISTENER ?
+                                    "OR" : "Dir");
+        log_warn(LD_CONFIG, "The IPv4 %sPort address %s does not match the "
+                 "descriptor address %s. If you have a static public IPv4 "
+                 "address, use 'Address <IPv4>' and 'OutboundBindAddress "
+                 "<IPv4>'. If you are behind a NAT, use two %sPort lines: "
+                 "'%sPort <PublicPort> NoListen' and '%sPort <InternalPort> "
+                 "NoAdvertise'.",
+                 listener_str, port_addr_str, desc_addr_str, listener_str,
+                 listener_str, listener_str);
+      }
+}
+
+/* Tor relays only have one IPv4 address in the descriptor, which is derived
+ * from the Address torrc option, or guessed using various methods in
+ * router_pick_published_address().
+ * Warn the operator if there is no ORPort on the descriptor address
+ * ipv4h_desc_addr.
+ * Warn the operator if there is no DirPort on the descriptor address.
+ * This catches a few common config errors:
+ *  - operators who expect ORPorts and DirPorts to be advertised on the
+ *    ports' listen addresses, rather than the torrc Address (or guessed
+ *    addresses in the absence of an Address config). This includes
+ *    operators who attempt to put their ORPort and DirPort on different
+ *    addresses;
+ *  - discrepancies between guessed addresses and configured listen
+ *    addresses (when the Address option isn't set).
+ * If a listener is listening on all IPv4 addresses, it is assumed that it
+ * is listening on the configured Address, and no messages are logged.
+ * If an operators has specified NoAdvertise ORPorts in a NAT setting,
+ * no messages are logged, unless they have specified other advertised
+ * addresses.
+ * The message tells operators to configure an ORPort and DirPort that match
+ * the Address (using NoListen if needed).
+ */
+static void
+router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr)
+{
+  router_check_descriptor_address_port_consistency(ipv4h_desc_addr,
+                                                   CONN_TYPE_OR_LISTENER);
+  router_check_descriptor_address_port_consistency(ipv4h_desc_addr,
+                                                   CONN_TYPE_DIR_LISTENER);
+}
+
+/** Build a fresh routerinfo, signed server descriptor, and extra-info document
+ * for this OR. Set r to the generated routerinfo, e to the generated
+ * extra-info document. Return 0 on success, -1 on temporary error. Failure to
+ * generate an extra-info document is not an error and is indicated by setting
+ * e to NULL. Caller is responsible for freeing generated documents if 0 is
+ * returned.
+ */
+int
+router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+{
+  routerinfo_t *ri;
+  extrainfo_t *ei;
+  uint32_t addr;
+  char platform[256];
+  int hibernating = we_are_hibernating();
+  const or_options_t *options = get_options();
+
+  if (router_pick_published_address(options, &addr, 0) < 0) {
+    log_warn(LD_CONFIG, "Don't know my address while generating descriptor");
+    return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+  }
+
+  /* Log a message if the address in the descriptor doesn't match the ORPort
+   * and DirPort addresses configured by the operator. */
+  router_check_descriptor_address_consistency(addr);
+
+  ri = tor_malloc_zero(sizeof(routerinfo_t));
+  ri->cache_info.routerlist_index = -1;
+  ri->nickname = tor_strdup(options->Nickname);
+  ri->addr = addr;
+  ri->or_port = router_get_advertised_or_port(options);
+  ri->dir_port = router_get_advertised_dir_port(options, 0);
+  ri->supports_tunnelled_dir_requests =
+    directory_permits_begindir_requests(options);
+  ri->cache_info.published_on = time(NULL);
+  ri->onion_pkey = crypto_pk_dup_key(get_onion_key()); /* must invoke from
+                                                        * main thread */
+  ri->onion_curve25519_pkey =
+    tor_memdup(&get_current_curve25519_keypair()->pubkey,
+               sizeof(curve25519_public_key_t));
+
+  /* For now, at most one IPv6 or-address is being advertised. */
+  {
+    const port_cfg_t *ipv6_orport = NULL;
+    SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
+      if (p->type == CONN_TYPE_OR_LISTENER &&
+          ! p->server_cfg.no_advertise &&
+          ! p->server_cfg.bind_ipv4_only &&
+          tor_addr_family(&p->addr) == AF_INET6) {
+        /* Like IPv4, if the relay is configured using the default
+         * authorities, disallow internal IPs. Otherwise, allow them. */
+        const int default_auth = using_default_dir_authorities(options);
+        if (! tor_addr_is_internal(&p->addr, 0) || ! default_auth) {
+          ipv6_orport = p;
+          break;
+        } else {
+          char addrbuf[TOR_ADDR_BUF_LEN];
+          log_warn(LD_CONFIG,
+                   "Unable to use configured IPv6 address \"%s\" in a "
+                   "descriptor. Skipping it. "
+                   "Try specifying a globally reachable address explicitly.",
+                   tor_addr_to_str(addrbuf, &p->addr, sizeof(addrbuf), 1));
+        }
+      }
+    } SMARTLIST_FOREACH_END(p);
+    if (ipv6_orport) {
+      tor_addr_copy(&ri->ipv6_addr, &ipv6_orport->addr);
+      ri->ipv6_orport = ipv6_orport->port;
+    }
+  }
+
+  ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key());
+  if (crypto_pk_get_digest(ri->identity_pkey,
+                           ri->cache_info.identity_digest)<0) {
+    routerinfo_free(ri);
+    return TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+  }
+  ri->cache_info.signing_key_cert =
+    tor_cert_dup(get_master_signing_key_cert());
+
+  get_platform_str(platform, sizeof(platform));
+  ri->platform = tor_strdup(platform);
+
+  ri->protocol_list = tor_strdup(protover_get_supported_protocols());
+
+  /* compute ri->bandwidthrate as the min of various options */
+  ri->bandwidthrate = get_effective_bwrate(options);
+
+  /* and compute ri->bandwidthburst similarly */
+  ri->bandwidthburst = get_effective_bwburst(options);
+
+  /* Report bandwidth, unless we're hibernating or shutting down */
+  ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess();
+
+  if (dns_seems_to_be_broken() || has_dns_init_failed()) {
+    /* DNS is screwed up; don't claim to be an exit. */
+    policies_exit_policy_append_reject_star(&ri->exit_policy);
+  } else {
+    policies_parse_exit_policy_from_options(options,ri->addr,&ri->ipv6_addr,
+                                            &ri->exit_policy);
+  }
+  ri->policy_is_reject_star =
+    policy_is_reject_star(ri->exit_policy, AF_INET, 1) &&
+    policy_is_reject_star(ri->exit_policy, AF_INET6, 1);
+
+  if (options->IPv6Exit) {
+    char *p_tmp = policy_summarize(ri->exit_policy, AF_INET6);
+    if (p_tmp)
+      ri->ipv6_exit_policy = parse_short_policy(p_tmp);
+    tor_free(p_tmp);
+  }
+
+  if (options->MyFamily && ! options->BridgeRelay) {
+    if (!warned_nonexistent_family)
+      warned_nonexistent_family = smartlist_new();
+    ri->declared_family = smartlist_new();
+    config_line_t *family;
+    for (family = options->MyFamily; family; family = family->next) {
+       char *name = family->value;
+       const node_t *member;
+       if (!strcasecmp(name, options->Nickname))
+         continue; /* Don't list ourself, that's redundant */
+       else
+         member = node_get_by_nickname(name, 0);
+       if (!member) {
+         int is_legal = is_legal_nickname_or_hexdigest(name);
+         if (!smartlist_contains_string(warned_nonexistent_family, name) &&
+             !is_legal_hexdigest(name)) {
+           if (is_legal)
+             log_warn(LD_CONFIG,
+                      "I have no descriptor for the router named \"%s\" in my "
+                      "declared family; I'll use the nickname as is, but "
+                      "this may confuse clients.", name);
+           else
+             log_warn(LD_CONFIG, "There is a router named \"%s\" in my "
+                      "declared family, but that isn't a legal nickname. "
+                      "Skipping it.", escaped(name));
+           smartlist_add_strdup(warned_nonexistent_family, name);
+         }
+         if (is_legal) {
+           smartlist_add_strdup(ri->declared_family, name);
+         }
+       } else if (router_digest_is_me(member->identity)) {
+         /* Don't list ourself in our own family; that's redundant */
+         /* XXX shouldn't be possible */
+       } else {
+         char *fp = tor_malloc(HEX_DIGEST_LEN+2);
+         fp[0] = '$';
+         base16_encode(fp+1,HEX_DIGEST_LEN+1,
+                       member->identity, DIGEST_LEN);
+         smartlist_add(ri->declared_family, fp);
+         if (smartlist_contains_string(warned_nonexistent_family, name))
+           smartlist_string_remove(warned_nonexistent_family, name);
+       }
+    }
+
+    /* remove duplicates from the list */
+    smartlist_sort_strings(ri->declared_family);
+    smartlist_uniq_strings(ri->declared_family);
+  }
+
+  /* Now generate the extrainfo. */
+  ei = tor_malloc_zero(sizeof(extrainfo_t));
+  ei->cache_info.is_extrainfo = 1;
+  strlcpy(ei->nickname, get_options()->Nickname, sizeof(ei->nickname));
+  ei->cache_info.published_on = ri->cache_info.published_on;
+  ei->cache_info.signing_key_cert =
+    tor_cert_dup(get_master_signing_key_cert());
+
+  memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest,
+         DIGEST_LEN);
+  if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body,
+                               ei, get_server_identity_key(),
+                               get_master_signing_keypair()) < 0) {
+    log_warn(LD_BUG, "Couldn't generate extra-info descriptor.");
+    extrainfo_free(ei);
+    ei = NULL;
+  } else {
+    ei->cache_info.signed_descriptor_len =
+      strlen(ei->cache_info.signed_descriptor_body);
+    router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
+                              ei->cache_info.signed_descriptor_len,
+                              ei->cache_info.signed_descriptor_digest);
+    crypto_digest256((char*) ei->digest256,
+                     ei->cache_info.signed_descriptor_body,
+                     ei->cache_info.signed_descriptor_len,
+                     DIGEST_SHA256);
+  }
+
+  /* Now finish the router descriptor. */
+  if (ei) {
+    memcpy(ri->cache_info.extra_info_digest,
+           ei->cache_info.signed_descriptor_digest,
+           DIGEST_LEN);
+    memcpy(ri->cache_info.extra_info_digest256,
+           ei->digest256,
+           DIGEST256_LEN);
+  } else {
+    /* ri was allocated with tor_malloc_zero, so there is no need to
+     * zero ri->cache_info.extra_info_digest here. */
+  }
+  if (! (ri->cache_info.signed_descriptor_body =
+          router_dump_router_to_string(ri, get_server_identity_key(),
+                                       get_onion_key(),
+                                       get_current_curve25519_keypair(),
+                                       get_master_signing_keypair())) ) {
+    log_warn(LD_BUG, "Couldn't generate router descriptor.");
+    routerinfo_free(ri);
+    extrainfo_free(ei);
+    return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
+  }
+  ri->cache_info.signed_descriptor_len =
+    strlen(ri->cache_info.signed_descriptor_body);
+
+  ri->purpose =
+    options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
+  if (options->BridgeRelay) {
+    /* Bridges shouldn't be able to send their descriptors unencrypted,
+       anyway, since they don't have a DirPort, and always connect to the
+       bridge authority anonymously.  But just in case they somehow think of
+       sending them on an unencrypted connection, don't allow them to try. */
+    ri->cache_info.send_unencrypted = 0;
+    if (ei)
+      ei->cache_info.send_unencrypted = 0;
+  } else {
+    ri->cache_info.send_unencrypted = 1;
+    if (ei)
+      ei->cache_info.send_unencrypted = 1;
+  }
+
+  router_get_router_hash(ri->cache_info.signed_descriptor_body,
+                         strlen(ri->cache_info.signed_descriptor_body),
+                         ri->cache_info.signed_descriptor_digest);
+
+  if (ei) {
+    tor_assert(!
+          routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
+                                                 &ri->cache_info, NULL));
+  }
+
+  *r = ri;
+  *e = ei;
+  return 0;
+}
+
+/** If <b>force</b> is true, or our descriptor is out-of-date, rebuild a fresh
+ * routerinfo, signed server descriptor, and extra-info document for this OR.
+ * Return 0 on success, -1 on temporary error.
+ */
+int
+router_rebuild_descriptor(int force)
+{
+  int err = 0;
+  routerinfo_t *ri;
+  extrainfo_t *ei;
+  uint32_t addr;
+  const or_options_t *options = get_options();
+
+  if (desc_clean_since && !force)
+    return 0;
+
+  if (router_pick_published_address(options, &addr, 0) < 0 ||
+      router_get_advertised_or_port(options) == 0) {
+    /* Stop trying to rebuild our descriptor every second. We'll
+     * learn that it's time to try again when ip_address_changed()
+     * marks it dirty. */
+    desc_clean_since = time(NULL);
+    return TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
+  }
+
+  log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : "");
+
+  err = router_build_fresh_descriptor(&ri, &ei);
+  if (err < 0) {
+    return err;
+  }
+
+  routerinfo_free(desc_routerinfo);
+  desc_routerinfo = ri;
+  extrainfo_free(desc_extrainfo);
+  desc_extrainfo = ei;
+
+  desc_clean_since = time(NULL);
+  desc_needs_upload = 1;
+  desc_gen_reason = desc_dirty_reason;
+  if (BUG(desc_gen_reason == NULL)) {
+    desc_gen_reason = "descriptor was marked dirty earlier, for no reason.";
+  }
+  desc_dirty_reason = NULL;
+  control_event_my_descriptor_changed();
+  return 0;
+}
+
+/** If our router descriptor ever goes this long without being regenerated
+ * because something changed, we force an immediate regenerate-and-upload. */
+#define FORCE_REGENERATE_DESCRIPTOR_INTERVAL (18*60*60)
+
+/** If our router descriptor seems to be missing or unacceptable according
+ * to the authorities, regenerate and reupload it _this_ often. */
+#define FAST_RETRY_DESCRIPTOR_INTERVAL (90*60)
+
+/** Mark descriptor out of date if it's been "too long" since we last tried
+ * to upload one. */
+void
+mark_my_descriptor_dirty_if_too_old(time_t now)
+{
+  networkstatus_t *ns;
+  const routerstatus_t *rs;
+  const char *retry_fast_reason = NULL; /* Set if we should retry frequently */
+  const time_t slow_cutoff = now - FORCE_REGENERATE_DESCRIPTOR_INTERVAL;
+  const time_t fast_cutoff = now - FAST_RETRY_DESCRIPTOR_INTERVAL;
+
+  /* If it's already dirty, don't mark it. */
+  if (! desc_clean_since)
+    return;
+
+  /* If it's older than FORCE_REGENERATE_DESCRIPTOR_INTERVAL, it's always
+   * time to rebuild it. */
+  if (desc_clean_since < slow_cutoff) {
+    mark_my_descriptor_dirty("time for new descriptor");
+    return;
+  }
+  /* Now we see whether we want to be retrying frequently or no.  The
+   * rule here is that we'll retry frequently if we aren't listed in the
+   * live consensus we have, or if the publication time of the
+   * descriptor listed for us in the consensus is very old. */
+  ns = networkstatus_get_live_consensus(now);
+  if (ns) {
+    rs = networkstatus_vote_find_entry(ns, server_identitykey_digest);
+    if (rs == NULL)
+      retry_fast_reason = "not listed in consensus";
+    else if (rs->published_on < slow_cutoff)
+      retry_fast_reason = "version listed in consensus is quite old";
+  }
+
+  if (retry_fast_reason && desc_clean_since < fast_cutoff)
+    mark_my_descriptor_dirty(retry_fast_reason);
+}
+
+/** Call when the current descriptor is out of date. */
+void
+mark_my_descriptor_dirty(const char *reason)
+{
+  const or_options_t *options = get_options();
+  if (BUG(reason == NULL)) {
+    reason = "marked descriptor dirty for unspecified reason";
+  }
+  if (server_mode(options) && options->PublishServerDescriptor_)
+    log_info(LD_OR, "Decided to publish new relay descriptor: %s", reason);
+  desc_clean_since = 0;
+  if (!desc_dirty_reason)
+    desc_dirty_reason = reason;
+}
+
+/** How frequently will we republish our descriptor because of large (factor
+ * of 2) shifts in estimated bandwidth? Note: We don't use this constant
+ * if our previous bandwidth estimate was exactly 0. */
+#define MAX_BANDWIDTH_CHANGE_FREQ (3*60*60)
+
+/** Check whether bandwidth has changed a lot since the last time we announced
+ * bandwidth. If so, mark our descriptor dirty. */
+void
+check_descriptor_bandwidth_changed(time_t now)
+{
+  static time_t last_changed = 0;
+  uint64_t prev, cur;
+  if (!router_get_my_routerinfo())
+    return;
+
+  prev = router_get_my_routerinfo()->bandwidthcapacity;
+  /* Consider ourselves to have zero bandwidth if we're hibernating or
+   * shutting down. */
+  cur = we_are_hibernating() ? 0 : rep_hist_bandwidth_assess();
+  if ((prev != cur && (!prev || !cur)) ||
+      cur > prev*2 ||
+      cur < prev/2) {
+    if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now || !prev) {
+      log_info(LD_GENERAL,
+               "Measured bandwidth has changed; rebuilding descriptor.");
+      mark_my_descriptor_dirty("bandwidth has changed");
+      last_changed = now;
+    }
+  }
+}
+
+/** Note at log level severity that our best guess of address has changed from
+ * <b>prev</b> to <b>cur</b>. */
+static void
+log_addr_has_changed(int severity,
+                     const tor_addr_t *prev,
+                     const tor_addr_t *cur,
+                     const char *source)
+{
+  char addrbuf_prev[TOR_ADDR_BUF_LEN];
+  char addrbuf_cur[TOR_ADDR_BUF_LEN];
+
+  if (tor_addr_to_str(addrbuf_prev, prev, sizeof(addrbuf_prev), 1) == NULL)
+    strlcpy(addrbuf_prev, "???", TOR_ADDR_BUF_LEN);
+  if (tor_addr_to_str(addrbuf_cur, cur, sizeof(addrbuf_cur), 1) == NULL)
+    strlcpy(addrbuf_cur, "???", TOR_ADDR_BUF_LEN);
+
+  if (!tor_addr_is_null(prev))
+    log_fn(severity, LD_GENERAL,
+           "Our IP Address has changed from %s to %s; "
+           "rebuilding descriptor (source: %s).",
+           addrbuf_prev, addrbuf_cur, source);
+  else
+    log_notice(LD_GENERAL,
+             "Guessed our IP address as %s (source: %s).",
+             addrbuf_cur, source);
+}
+
+/** Check whether our own address as defined by the Address configuration
+ * has changed. This is for routers that get their address from a service
+ * like dyndns. If our address has changed, mark our descriptor dirty. */
+void
+check_descriptor_ipaddress_changed(time_t now)
+{
+  uint32_t prev, cur;
+  const or_options_t *options = get_options();
+  const char *method = NULL;
+  char *hostname = NULL;
+
+  (void) now;
+
+  if (router_get_my_routerinfo() == NULL)
+    return;
+
+  /* XXXX ipv6 */
+  prev = router_get_my_routerinfo()->addr;
+  if (resolve_my_address(LOG_INFO, options, &cur, &method, &hostname) < 0) {
+    log_info(LD_CONFIG,"options->Address didn't resolve into an IP.");
+    return;
+  }
+
+  if (prev != cur) {
+    char *source;
+    tor_addr_t tmp_prev, tmp_cur;
+
+    tor_addr_from_ipv4h(&tmp_prev, prev);
+    tor_addr_from_ipv4h(&tmp_cur, cur);
+
+    tor_asprintf(&source, "METHOD=%s%s%s", method,
+                 hostname ? " HOSTNAME=" : "",
+                 hostname ? hostname : "");
+
+    log_addr_has_changed(LOG_NOTICE, &tmp_prev, &tmp_cur, source);
+    tor_free(source);
+
+    ip_address_changed(0);
+  }
+
+  tor_free(hostname);
+}
+
+/** The most recently guessed value of our IP address, based on directory
+ * headers. */
+static tor_addr_t last_guessed_ip = TOR_ADDR_NULL;
+
+/** A directory server <b>d_conn</b> told us our IP address is
+ * <b>suggestion</b>.
+ * If this address is different from the one we think we are now, and
+ * if our computer doesn't actually know its IP address, then switch. */
+void
+router_new_address_suggestion(const char *suggestion,
+                              const dir_connection_t *d_conn)
+{
+  tor_addr_t addr;
+  uint32_t cur = 0;             /* Current IPv4 address.  */
+  const or_options_t *options = get_options();
+
+  /* first, learn what the IP address actually is */
+  if (tor_addr_parse(&addr, suggestion) == -1) {
+    log_debug(LD_DIR, "Malformed X-Your-Address-Is header %s. Ignoring.",
+              escaped(suggestion));
+    return;
+  }
+
+  log_debug(LD_DIR, "Got X-Your-Address-Is: %s.", suggestion);
+
+  if (!server_mode(options)) {
+    tor_addr_copy(&last_guessed_ip, &addr);
+    return;
+  }
+
+  /* XXXX ipv6 */
+  cur = get_last_resolved_addr();
+  if (cur ||
+      resolve_my_address(LOG_INFO, options, &cur, NULL, NULL) >= 0) {
+    /* We're all set -- we already know our address. Great. */
+    tor_addr_from_ipv4h(&last_guessed_ip, cur); /* store it in case we
+                                                   need it later */
+    return;
+  }
+  if (tor_addr_is_internal(&addr, 0)) {
+    /* Don't believe anybody who says our IP is, say, 127.0.0.1. */
+    return;
+  }
+  if (tor_addr_eq(&d_conn->base_.addr, &addr)) {
+    /* Don't believe anybody who says our IP is their IP. */
+    log_debug(LD_DIR, "A directory server told us our IP address is %s, "
+              "but they are just reporting their own IP address. Ignoring.",
+              suggestion);
+    return;
+  }
+
+  /* Okay.  We can't resolve our own address, and X-Your-Address-Is is giving
+   * us an answer different from what we had the last time we managed to
+   * resolve it. */
+  if (!tor_addr_eq(&last_guessed_ip, &addr)) {
+    control_event_server_status(LOG_NOTICE,
+                                "EXTERNAL_ADDRESS ADDRESS=%s METHOD=DIRSERV",
+                                suggestion);
+    log_addr_has_changed(LOG_NOTICE, &last_guessed_ip, &addr,
+                         d_conn->base_.address);
+    ip_address_changed(0);
+    tor_addr_copy(&last_guessed_ip, &addr); /* router_rebuild_descriptor()
+                                               will fetch it */
+  }
+}
+
+/** We failed to resolve our address locally, but we'd like to build
+ * a descriptor and publish / test reachability. If we have a guess
+ * about our address based on directory headers, answer it and return
+ * 0; else return -1. */
+static int
+router_guess_address_from_dir_headers(uint32_t *guess)
+{
+  if (!tor_addr_is_null(&last_guessed_ip)) {
+    *guess = tor_addr_to_ipv4h(&last_guessed_ip);
+    return 0;
+  }
+  return -1;
+}
+
+/** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short
+ * string describing the version of Tor and the operating system we're
+ * currently running on.
+ */
+STATIC void
+get_platform_str(char *platform, size_t len)
+{
+  tor_snprintf(platform, len, "Tor %s on %s",
+               get_short_version(), get_uname());
+}
+
+/* XXX need to audit this thing and count fenceposts. maybe
+ *     refactor so we don't have to keep asking if we're
+ *     near the end of maxlen?
+ */
+#define DEBUG_ROUTER_DUMP_ROUTER_TO_STRING
+
+/** OR only: Given a routerinfo for this router, and an identity key to sign
+ * with, encode the routerinfo as a signed server descriptor and return a new
+ * string encoding the result, or NULL on failure.
+ */
+char *
+router_dump_router_to_string(routerinfo_t *router,
+                             const crypto_pk_t *ident_key,
+                             const crypto_pk_t *tap_key,
+                             const curve25519_keypair_t *ntor_keypair,
+                             const ed25519_keypair_t *signing_keypair)
+{
+  char *address = NULL;
+  char *onion_pkey = NULL; /* Onion key, PEM-encoded. */
+  char *identity_pkey = NULL; /* Identity key, PEM-encoded. */
+  char digest[DIGEST256_LEN];
+  char published[ISO_TIME_LEN+1];
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *extra_info_line = NULL;
+  size_t onion_pkeylen, identity_pkeylen;
+  char *family_line = NULL;
+  char *extra_or_address = NULL;
+  const or_options_t *options = get_options();
+  smartlist_t *chunks = NULL;
+  char *output = NULL;
+  const int emit_ed_sigs = signing_keypair &&
+    router->cache_info.signing_key_cert;
+  char *ed_cert_line = NULL;
+  char *rsa_tap_cc_line = NULL;
+  char *ntor_cc_line = NULL;
+  char *proto_line = NULL;
+
+  /* Make sure the identity key matches the one in the routerinfo. */
+  if (!crypto_pk_eq_keys(ident_key, router->identity_pkey)) {
+    log_warn(LD_BUG,"Tried to sign a router with a private key that didn't "
+             "match router's public key!");
+    goto err;
+  }
+  if (emit_ed_sigs) {
+    if (!router->cache_info.signing_key_cert->signing_key_included ||
+        !ed25519_pubkey_eq(&router->cache_info.signing_key_cert->signed_key,
+                           &signing_keypair->pubkey)) {
+      log_warn(LD_BUG, "Tried to sign a router descriptor with a mismatched "
+               "ed25519 key chain %d",
+               router->cache_info.signing_key_cert->signing_key_included);
+      goto err;
+    }
+  }
+
+  /* record our fingerprint, so we can include it in the descriptor */
+  if (crypto_pk_get_fingerprint(router->identity_pkey, fingerprint, 1)<0) {
+    log_err(LD_BUG,"Error computing fingerprint");
+    goto err;
+  }
+
+  if (emit_ed_sigs) {
+    /* Encode ed25519 signing cert */
+    char ed_cert_base64[256];
+    char ed_fp_base64[ED25519_BASE64_LEN+1];
+    if (base64_encode(ed_cert_base64, sizeof(ed_cert_base64),
+                    (const char*)router->cache_info.signing_key_cert->encoded,
+                    router->cache_info.signing_key_cert->encoded_len,
+                    BASE64_ENCODE_MULTILINE) < 0) {
+      log_err(LD_BUG,"Couldn't base64-encode signing key certificate!");
+      goto err;
+    }
+    if (ed25519_public_to_base64(ed_fp_base64,
+                       &router->cache_info.signing_key_cert->signing_key)<0) {
+      log_err(LD_BUG,"Couldn't base64-encode identity key\n");
+      goto err;
+    }
+    tor_asprintf(&ed_cert_line, "identity-ed25519\n"
+                 "-----BEGIN ED25519 CERT-----\n"
+                 "%s"
+                 "-----END ED25519 CERT-----\n"
+                 "master-key-ed25519 %s\n",
+                 ed_cert_base64, ed_fp_base64);
+  }
+
+  /* PEM-encode the onion key */
+  if (crypto_pk_write_public_key_to_string(router->onion_pkey,
+                                           &onion_pkey,&onion_pkeylen)<0) {
+    log_warn(LD_BUG,"write onion_pkey to string failed!");
+    goto err;
+  }
+
+  /* PEM-encode the identity key */
+  if (crypto_pk_write_public_key_to_string(router->identity_pkey,
+                                        &identity_pkey,&identity_pkeylen)<0) {
+    log_warn(LD_BUG,"write identity_pkey to string failed!");
+    goto err;
+  }
+
+  /* Cross-certify with RSA key */
+  if (tap_key && router->cache_info.signing_key_cert &&
+      router->cache_info.signing_key_cert->signing_key_included) {
+    char buf[256];
+    int tap_cc_len = 0;
+    uint8_t *tap_cc =
+      make_tap_onion_key_crosscert(tap_key,
+                            &router->cache_info.signing_key_cert->signing_key,
+                            router->identity_pkey,
+                            &tap_cc_len);
+    if (!tap_cc) {
+      log_warn(LD_BUG,"make_tap_onion_key_crosscert failed!");
+      goto err;
+    }
+
+    if (base64_encode(buf, sizeof(buf), (const char*)tap_cc, tap_cc_len,
+                      BASE64_ENCODE_MULTILINE) < 0) {
+      log_warn(LD_BUG,"base64_encode(rsa_crosscert) failed!");
+      tor_free(tap_cc);
+      goto err;
+    }
+    tor_free(tap_cc);
+
+    tor_asprintf(&rsa_tap_cc_line,
+                 "onion-key-crosscert\n"
+                 "-----BEGIN CROSSCERT-----\n"
+                 "%s"
+                 "-----END CROSSCERT-----\n", buf);
+  }
+
+  /* Cross-certify with onion keys */
+  if (ntor_keypair && router->cache_info.signing_key_cert &&
+      router->cache_info.signing_key_cert->signing_key_included) {
+    int sign = 0;
+    char buf[256];
+    /* XXXX Base the expiration date on the actual onion key expiration time?*/
+    tor_cert_t *cert =
+      make_ntor_onion_key_crosscert(ntor_keypair,
+                         &router->cache_info.signing_key_cert->signing_key,
+                         router->cache_info.published_on,
+                         get_onion_key_lifetime(), &sign);
+    if (!cert) {
+      log_warn(LD_BUG,"make_ntor_onion_key_crosscert failed!");
+      goto err;
+    }
+    tor_assert(sign == 0 || sign == 1);
+
+    if (base64_encode(buf, sizeof(buf),
+                      (const char*)cert->encoded, cert->encoded_len,
+                      BASE64_ENCODE_MULTILINE)<0) {
+      log_warn(LD_BUG,"base64_encode(ntor_crosscert) failed!");
+      tor_cert_free(cert);
+      goto err;
+    }
+    tor_cert_free(cert);
+
+    tor_asprintf(&ntor_cc_line,
+                 "ntor-onion-key-crosscert %d\n"
+                 "-----BEGIN ED25519 CERT-----\n"
+                 "%s"
+                 "-----END ED25519 CERT-----\n", sign, buf);
+  }
+
+  /* Encode the publication time. */
+  format_iso_time(published, router->cache_info.published_on);
+
+  if (router->declared_family && smartlist_len(router->declared_family)) {
+    char *family = smartlist_join_strings(router->declared_family,
+                                          " ", 0, NULL);
+    tor_asprintf(&family_line, "family %s\n", family);
+    tor_free(family);
+  } else {
+    family_line = tor_strdup("");
+  }
+
+  if (!tor_digest_is_zero(router->cache_info.extra_info_digest)) {
+    char extra_info_digest[HEX_DIGEST_LEN+1];
+    base16_encode(extra_info_digest, sizeof(extra_info_digest),
+                  router->cache_info.extra_info_digest, DIGEST_LEN);
+    if (!tor_digest256_is_zero(router->cache_info.extra_info_digest256)) {
+      char d256_64[BASE64_DIGEST256_LEN+1];
+      digest256_to_base64(d256_64, router->cache_info.extra_info_digest256);
+      tor_asprintf(&extra_info_line, "extra-info-digest %s %s\n",
+                   extra_info_digest, d256_64);
+    } else {
+      tor_asprintf(&extra_info_line, "extra-info-digest %s\n",
+                   extra_info_digest);
+    }
+  }
+
+  if (router->ipv6_orport &&
+      tor_addr_family(&router->ipv6_addr) == AF_INET6) {
+    char addr[TOR_ADDR_BUF_LEN];
+    const char *a;
+    a = tor_addr_to_str(addr, &router->ipv6_addr, sizeof(addr), 1);
+    if (a) {
+      tor_asprintf(&extra_or_address,
+                   "or-address %s:%d\n", a, router->ipv6_orport);
+      log_debug(LD_OR, "My or-address line is <%s>", extra_or_address);
+    }
+  }
+
+  if (router->protocol_list) {
+    tor_asprintf(&proto_line, "proto %s\n", router->protocol_list);
+  } else {
+    proto_line = tor_strdup("");
+  }
+
+  address = tor_dup_ip(router->addr);
+  chunks = smartlist_new();
+
+  /* Generate the easy portion of the router descriptor. */
+  smartlist_add_asprintf(chunks,
+                    "router %s %s %d 0 %d\n"
+                    "%s"
+                    "%s"
+                    "platform %s\n"
+                    "%s"
+                    "published %s\n"
+                    "fingerprint %s\n"
+                    "uptime %ld\n"
+                    "bandwidth %d %d %d\n"
+                    "%s%s"
+                    "onion-key\n%s"
+                    "signing-key\n%s"
+                    "%s%s"
+                    "%s%s%s",
+    router->nickname,
+    address,
+    router->or_port,
+    router_should_advertise_dirport(options, router->dir_port),
+    ed_cert_line ? ed_cert_line : "",
+    extra_or_address ? extra_or_address : "",
+    router->platform,
+    proto_line,
+    published,
+    fingerprint,
+    get_uptime(),
+    (int) router->bandwidthrate,
+    (int) router->bandwidthburst,
+    (int) router->bandwidthcapacity,
+    extra_info_line ? extra_info_line : "",
+    (options->DownloadExtraInfo || options->V3AuthoritativeDir) ?
+                         "caches-extra-info\n" : "",
+    onion_pkey, identity_pkey,
+    rsa_tap_cc_line ? rsa_tap_cc_line : "",
+    ntor_cc_line ? ntor_cc_line : "",
+    family_line,
+    we_are_hibernating() ? "hibernating 1\n" : "",
+    "hidden-service-dir\n");
+
+  if (options->ContactInfo && strlen(options->ContactInfo)) {
+    const char *ci = options->ContactInfo;
+    if (strchr(ci, '\n') || strchr(ci, '\r'))
+      ci = escaped(ci);
+    smartlist_add_asprintf(chunks, "contact %s\n", ci);
+  }
+
+  if (options->BridgeRelay) {
+    const char *bd;
+    if (options->BridgeDistribution && strlen(options->BridgeDistribution)) {
+      bd = options->BridgeDistribution;
+    } else {
+      bd = "any";
+    }
+    if (strchr(bd, '\n') || strchr(bd, '\r'))
+      bd = escaped(bd);
+    smartlist_add_asprintf(chunks, "bridge-distribution-request %s\n", bd);
+  }
+
+  if (router->onion_curve25519_pkey) {
+    char kbuf[128];
+    base64_encode(kbuf, sizeof(kbuf),
+                  (const char *)router->onion_curve25519_pkey->public_key,
+                  CURVE25519_PUBKEY_LEN, BASE64_ENCODE_MULTILINE);
+    smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf);
+  } else {
+    /* Authorities will start rejecting relays without ntor keys in 0.2.9 */
+    log_err(LD_BUG, "A relay must have an ntor onion key");
+    goto err;
+  }
+
+  /* Write the exit policy to the end of 's'. */
+  if (!router->exit_policy || !smartlist_len(router->exit_policy)) {
+    smartlist_add_strdup(chunks, "reject *:*\n");
+  } else if (router->exit_policy) {
+    char *exit_policy = router_dump_exit_policy_to_string(router,1,0);
+
+    if (!exit_policy)
+      goto err;
+
+    smartlist_add_asprintf(chunks, "%s\n", exit_policy);
+    tor_free(exit_policy);
+  }
+
+  if (router->ipv6_exit_policy) {
+    char *p6 = write_short_policy(router->ipv6_exit_policy);
+    if (p6 && strcmp(p6, "reject 1-65535")) {
+      smartlist_add_asprintf(chunks,
+                            "ipv6-policy %s\n", p6);
+    }
+    tor_free(p6);
+  }
+
+  if (router_should_advertise_begindir(options,
+                                   router->supports_tunnelled_dir_requests)) {
+    smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
+  }
+
+  /* Sign the descriptor with Ed25519 */
+  if (emit_ed_sigs)  {
+    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
+    crypto_digest_smartlist_prefix(digest, DIGEST256_LEN,
+                                   ED_DESC_SIGNATURE_PREFIX,
+                                   chunks, "", DIGEST_SHA256);
+    ed25519_signature_t sig;
+    char buf[ED25519_SIG_BASE64_LEN+1];
+    if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN,
+                     signing_keypair) < 0)
+      goto err;
+    if (ed25519_signature_to_base64(buf, &sig) < 0)
+      goto err;
+
+    smartlist_add_asprintf(chunks, "%s\n", buf);
+  }
+
+  /* Sign the descriptor with RSA */
+  smartlist_add_strdup(chunks, "router-signature\n");
+
+  crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1);
+
+  {
+    char *sig;
+    if (!(sig = router_get_dirobj_signature(digest, DIGEST_LEN, ident_key))) {
+      log_warn(LD_BUG, "Couldn't sign router descriptor");
+      goto err;
+    }
+    smartlist_add(chunks, sig);
+  }
+
+  /* include a last '\n' */
+  smartlist_add_strdup(chunks, "\n");
+
+  output = smartlist_join_strings(chunks, "", 0, NULL);
+
+#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING
+  {
+    char *s_dup;
+    const char *cp;
+    routerinfo_t *ri_tmp;
+    cp = s_dup = tor_strdup(output);
+    ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0, NULL, NULL);
+    if (!ri_tmp) {
+      log_err(LD_BUG,
+              "We just generated a router descriptor we can't parse.");
+      log_err(LD_BUG, "Descriptor was: <<%s>>", output);
+      goto err;
+    }
+    tor_free(s_dup);
+    routerinfo_free(ri_tmp);
+  }
+#endif /* defined(DEBUG_ROUTER_DUMP_ROUTER_TO_STRING) */
+
+  goto done;
+
+ err:
+  tor_free(output); /* sets output to NULL */
+ done:
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+    smartlist_free(chunks);
+  }
+  tor_free(address);
+  tor_free(family_line);
+  tor_free(onion_pkey);
+  tor_free(identity_pkey);
+  tor_free(extra_or_address);
+  tor_free(ed_cert_line);
+  tor_free(rsa_tap_cc_line);
+  tor_free(ntor_cc_line);
+  tor_free(extra_info_line);
+  tor_free(proto_line);
+
+  return output;
+}
+
+/**
+ * OR only: Given <b>router</b>, produce a string with its exit policy.
+ * If <b>include_ipv4</b> is true, include IPv4 entries.
+ * If <b>include_ipv6</b> is true, include IPv6 entries.
+ */
+char *
+router_dump_exit_policy_to_string(const routerinfo_t *router,
+                                  int include_ipv4,
+                                  int include_ipv6)
+{
+  if ((!router->exit_policy) || (router->policy_is_reject_star)) {
+    return tor_strdup("reject *:*");
+  }
+
+  return policy_dump_to_string(router->exit_policy,
+                               include_ipv4,
+                               include_ipv6);
+}
+
+/** Copy the primary (IPv4) OR port (IP address and TCP port) for
+ * <b>router</b> into *<b>ap_out</b>. */
+void
+router_get_prim_orport(const routerinfo_t *router, tor_addr_port_t *ap_out)
+{
+  tor_assert(ap_out != NULL);
+  tor_addr_from_ipv4h(&ap_out->addr, router->addr);
+  ap_out->port = router->or_port;
+}
+
+/** Return 1 if any of <b>router</b>'s addresses are <b>addr</b>.
+ *   Otherwise return 0. */
+int
+router_has_addr(const routerinfo_t *router, const tor_addr_t *addr)
+{
+  return
+    tor_addr_eq_ipv4h(addr, router->addr) ||
+    tor_addr_eq(&router->ipv6_addr, addr);
+}
+
+int
+router_has_orport(const routerinfo_t *router, const tor_addr_port_t *orport)
+{
+  return
+    (tor_addr_eq_ipv4h(&orport->addr, router->addr) &&
+     orport->port == router->or_port) ||
+    (tor_addr_eq(&orport->addr, &router->ipv6_addr) &&
+     orport->port == router->ipv6_orport);
+}
+
+/** Load the contents of <b>filename</b>, find the last line starting with
+ * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in
+ * the past or more than 1 hour in the future with respect to <b>now</b>,
+ * and write the file contents starting with that line to *<b>out</b>.
+ * Return 1 for success, 0 if the file does not exist or is empty, or -1
+ * if the file does not contain a line matching these criteria or other
+ * failure. */
+static int
+load_stats_file(const char *filename, const char *end_line, time_t now,
+                char **out)
+{
+  int r = -1;
+  char *fname = get_datadir_fname(filename);
+  char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1];
+  time_t written;
+  switch (file_status(fname)) {
+    case FN_FILE:
+      /* X022 Find an alternative to reading the whole file to memory. */
+      if ((contents = read_file_to_str(fname, 0, NULL))) {
+        tmp = strstr(contents, end_line);
+        /* Find last block starting with end_line */
+        while (tmp) {
+          start = tmp;
+          tmp = strstr(tmp + 1, end_line);
+        }
+        if (!start)
+          goto notfound;
+        if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr))
+          goto notfound;
+        strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr));
+        if (parse_iso_time(timestr, &written) < 0)
+          goto notfound;
+        if (written < now - (25*60*60) || written > now + (1*60*60))
+          goto notfound;
+        *out = tor_strdup(start);
+        r = 1;
+      }
+     notfound:
+      tor_free(contents);
+      break;
+    /* treat empty stats files as if the file doesn't exist */
+    case FN_NOENT:
+    case FN_EMPTY:
+      r = 0;
+      break;
+    case FN_ERROR:
+    case FN_DIR:
+    default:
+      break;
+  }
+  tor_free(fname);
+  return r;
+}
+
+/** Write the contents of <b>extrainfo</b> and aggregated statistics to
+ * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on
+ * success, negative on failure. */
+int
+extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
+                         crypto_pk_t *ident_key,
+                         const ed25519_keypair_t *signing_keypair)
+{
+  const or_options_t *options = get_options();
+  char identity[HEX_DIGEST_LEN+1];
+  char published[ISO_TIME_LEN+1];
+  char digest[DIGEST_LEN];
+  char *bandwidth_usage;
+  int result;
+  static int write_stats_to_extrainfo = 1;
+  char sig[DIROBJ_MAX_SIG_LEN+1];
+  char *s = NULL, *pre, *contents, *cp, *s_dup = NULL;
+  time_t now = time(NULL);
+  smartlist_t *chunks = smartlist_new();
+  extrainfo_t *ei_tmp = NULL;
+  const int emit_ed_sigs = signing_keypair &&
+    extrainfo->cache_info.signing_key_cert;
+  char *ed_cert_line = NULL;
+
+  base16_encode(identity, sizeof(identity),
+                extrainfo->cache_info.identity_digest, DIGEST_LEN);
+  format_iso_time(published, extrainfo->cache_info.published_on);
+  bandwidth_usage = rep_hist_get_bandwidth_lines();
+  if (emit_ed_sigs) {
+    if (!extrainfo->cache_info.signing_key_cert->signing_key_included ||
+        !ed25519_pubkey_eq(&extrainfo->cache_info.signing_key_cert->signed_key,
+                           &signing_keypair->pubkey)) {
+      log_warn(LD_BUG, "Tried to sign a extrainfo descriptor with a "
+               "mismatched ed25519 key chain %d",
+               extrainfo->cache_info.signing_key_cert->signing_key_included);
+      goto err;
+    }
+    char ed_cert_base64[256];
+    if (base64_encode(ed_cert_base64, sizeof(ed_cert_base64),
+                 (const char*)extrainfo->cache_info.signing_key_cert->encoded,
+                 extrainfo->cache_info.signing_key_cert->encoded_len,
+                 BASE64_ENCODE_MULTILINE) < 0) {
+      log_err(LD_BUG,"Couldn't base64-encode signing key certificate!");
+      goto err;
+    }
+    tor_asprintf(&ed_cert_line, "identity-ed25519\n"
+                 "-----BEGIN ED25519 CERT-----\n"
+                 "%s"
+                 "-----END ED25519 CERT-----\n", ed_cert_base64);
+  } else {
+    ed_cert_line = tor_strdup("");
+  }
+
+  tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n%s",
+               extrainfo->nickname, identity,
+               ed_cert_line,
+               published, bandwidth_usage);
+  smartlist_add(chunks, pre);
+
+  if (geoip_is_loaded(AF_INET))
+    smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
+                           geoip_db_digest(AF_INET));
+  if (geoip_is_loaded(AF_INET6))
+    smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
+                           geoip_db_digest(AF_INET6));
+
+  if (options->ExtraInfoStatistics && write_stats_to_extrainfo) {
+    log_info(LD_GENERAL, "Adding stats to extra-info descriptor.");
+    if (options->DirReqStatistics &&
+        load_stats_file("stats"PATH_SEPARATOR"dirreq-stats",
+                        "dirreq-stats-end", now, &contents) > 0) {
+      smartlist_add(chunks, contents);
+    }
+    if (options->HiddenServiceStatistics &&
+        load_stats_file("stats"PATH_SEPARATOR"hidserv-stats",
+                        "hidserv-stats-end", now, &contents) > 0) {
+      smartlist_add(chunks, contents);
+    }
+    if (options->EntryStatistics &&
+        load_stats_file("stats"PATH_SEPARATOR"entry-stats",
+                        "entry-stats-end", now, &contents) > 0) {
+      smartlist_add(chunks, contents);
+    }
+    if (options->CellStatistics &&
+        load_stats_file("stats"PATH_SEPARATOR"buffer-stats",
+                        "cell-stats-end", now, &contents) > 0) {
+      smartlist_add(chunks, contents);
+    }
+    if (options->ExitPortStatistics &&
+        load_stats_file("stats"PATH_SEPARATOR"exit-stats",
+                        "exit-stats-end", now, &contents) > 0) {
+      smartlist_add(chunks, contents);
+    }
+    if (options->ConnDirectionStatistics &&
+        load_stats_file("stats"PATH_SEPARATOR"conn-stats",
+                        "conn-bi-direct", now, &contents) > 0) {
+      smartlist_add(chunks, contents);
+    }
+  }
+
+  if (options->PaddingStatistics) {
+    contents = rep_hist_get_padding_count_lines();
+    if (contents)
+      smartlist_add(chunks, contents);
+  }
+
+  /* Add information about the pluggable transports we support. */
+  if (options->ServerTransportPlugin) {
+    char *pluggable_transports = pt_get_extra_info_descriptor_string();
+    if (pluggable_transports)
+      smartlist_add(chunks, pluggable_transports);
+  }
+
+  if (should_record_bridge_info(options) && write_stats_to_extrainfo) {
+    const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
+    if (bridge_stats) {
+      smartlist_add_strdup(chunks, bridge_stats);
+    }
+  }
+
+  if (emit_ed_sigs) {
+    char sha256_digest[DIGEST256_LEN];
+    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
+    crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
+                                   ED_DESC_SIGNATURE_PREFIX,
+                                   chunks, "", DIGEST_SHA256);
+    ed25519_signature_t ed_sig;
+    char buf[ED25519_SIG_BASE64_LEN+1];
+    if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
+                     signing_keypair) < 0)
+      goto err;
+    if (ed25519_signature_to_base64(buf, &ed_sig) < 0)
+      goto err;
+
+    smartlist_add_asprintf(chunks, "%s\n", buf);
+  }
+
+  smartlist_add_strdup(chunks, "router-signature\n");
+  s = smartlist_join_strings(chunks, "", 0, NULL);
+
+  while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) {
+    /* So long as there are at least two chunks (one for the initial
+     * extra-info line and one for the router-signature), we can keep removing
+     * things. */
+    if (smartlist_len(chunks) > 2) {
+      /* We remove the next-to-last element (remember, len-1 is the last
+         element), since we need to keep the router-signature element. */
+      int idx = smartlist_len(chunks) - 2;
+      char *e = smartlist_get(chunks, idx);
+      smartlist_del_keeporder(chunks, idx);
+      log_warn(LD_GENERAL, "We just generated an extra-info descriptor "
+                           "with statistics that exceeds the 50 KB "
+                           "upload limit. Removing last added "
+                           "statistics.");
+      tor_free(e);
+      tor_free(s);
+      s = smartlist_join_strings(chunks, "", 0, NULL);
+    } else {
+      log_warn(LD_BUG, "We just generated an extra-info descriptors that "
+                       "exceeds the 50 KB upload limit.");
+      goto err;
+    }
+  }
+
+  memset(sig, 0, sizeof(sig));
+  if (router_get_extrainfo_hash(s, strlen(s), digest) < 0 ||
+      router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
+                                     ident_key) < 0) {
+    log_warn(LD_BUG, "Could not append signature to extra-info "
+                     "descriptor.");
+    goto err;
+  }
+  smartlist_add_strdup(chunks, sig);
+  tor_free(s);
+  s = smartlist_join_strings(chunks, "", 0, NULL);
+
+  cp = s_dup = tor_strdup(s);
+  ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL, NULL);
+  if (!ei_tmp) {
+    if (write_stats_to_extrainfo) {
+      log_warn(LD_GENERAL, "We just generated an extra-info descriptor "
+                           "with statistics that we can't parse. Not "
+                           "adding statistics to this or any future "
+                           "extra-info descriptors.");
+      write_stats_to_extrainfo = 0;
+      result = extrainfo_dump_to_string(s_out, extrainfo, ident_key,
+                                        signing_keypair);
+      goto done;
+    } else {
+      log_warn(LD_BUG, "We just generated an extrainfo descriptor we "
+                       "can't parse.");
+      goto err;
+    }
+  }
+
+  *s_out = s;
+  s = NULL; /* prevent free */
+  result = 0;
+  goto done;
+
+ err:
+  result = -1;
+
+ done:
+  tor_free(s);
+  SMARTLIST_FOREACH(chunks, char *, chunk, tor_free(chunk));
+  smartlist_free(chunks);
+  tor_free(s_dup);
+  tor_free(ed_cert_line);
+  extrainfo_free(ei_tmp);
+  tor_free(bandwidth_usage);
+
+  return result;
+}
+
+/** Return true iff <b>s</b> is a valid server nickname. (That is, a string
+ * containing between 1 and MAX_NICKNAME_LEN characters from
+ * LEGAL_NICKNAME_CHARACTERS.) */
+int
+is_legal_nickname(const char *s)
+{
+  size_t len;
+  tor_assert(s);
+  len = strlen(s);
+  return len > 0 && len <= MAX_NICKNAME_LEN &&
+    strspn(s,LEGAL_NICKNAME_CHARACTERS) == len;
+}
+
+/** Return true iff <b>s</b> is a valid server nickname or
+ * hex-encoded identity-key digest. */
+int
+is_legal_nickname_or_hexdigest(const char *s)
+{
+  if (*s!='$')
+    return is_legal_nickname(s);
+  else
+    return is_legal_hexdigest(s);
+}
+
+/** Return true iff <b>s</b> is a valid hex-encoded identity-key
+ * digest. (That is, an optional $, followed by 40 hex characters,
+ * followed by either nothing, or = or ~ followed by a nickname, or
+ * a character other than =, ~, or a hex character.)
+ */
+int
+is_legal_hexdigest(const char *s)
+{
+  size_t len;
+  tor_assert(s);
+  if (s[0] == '$') s++;
+  len = strlen(s);
+  if (len > HEX_DIGEST_LEN) {
+    if (s[HEX_DIGEST_LEN] == '=' ||
+        s[HEX_DIGEST_LEN] == '~') {
+      if (!is_legal_nickname(s+HEX_DIGEST_LEN+1))
+        return 0;
+    } else {
+      return 0;
+    }
+  }
+  return (len >= HEX_DIGEST_LEN &&
+          strspn(s,HEX_CHARACTERS)==HEX_DIGEST_LEN);
+}
+
+/**
+ * Longest allowed output of format_node_description, plus 1 character for
+ * NUL.  This allows space for:
+ * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
+ * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
+ * plus a terminating NUL.
+ */
+#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
+
+/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
+ * hold a human-readable description of a node with identity digest
+ * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>,
+ * and address <b>addr</b> or <b>addr32h</b>.
+ *
+ * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to
+ * NULL.  The <b>addr32h</b> field is optional and may be set to 0.
+ *
+ * Return a pointer to the front of <b>buf</b>.
+ */
+const char *
+format_node_description(char *buf,
+                        const char *id_digest,
+                        int is_named,
+                        const char *nickname,
+                        const tor_addr_t *addr,
+                        uint32_t addr32h)
+{
+  char *cp;
+
+  if (!buf)
+    return "<NULL BUFFER>";
+
+  buf[0] = '$';
+  base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
+  cp = buf+1+HEX_DIGEST_LEN;
+  if (nickname) {
+    buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
+    strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
+    cp += strlen(cp);
+  }
+  if (addr32h || addr) {
+    memcpy(cp, " at ", 4);
+    cp += 4;
+    if (addr) {
+      tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0);
+    } else {
+      struct in_addr in;
+      in.s_addr = htonl(addr32h);
+      tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN);
+    }
+  }
+  return buf;
+}
+
+/** Return a human-readable description of the routerinfo_t <b>ri</b>.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+router_describe(const routerinfo_t *ri)
+{
+  static char buf[NODE_DESC_BUF_LEN];
+
+  if (!ri)
+    return "<null>";
+  return format_node_description(buf,
+                                 ri->cache_info.identity_digest,
+                                 0,
+                                 ri->nickname,
+                                 NULL,
+                                 ri->addr);
+}
+
+/** Return a human-readable description of the node_t <b>node</b>.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+node_describe(const node_t *node)
+{
+  static char buf[NODE_DESC_BUF_LEN];
+  const char *nickname = NULL;
+  uint32_t addr32h = 0;
+  int is_named = 0;
+
+  if (!node)
+    return "<null>";
+
+  if (node->rs) {
+    nickname = node->rs->nickname;
+    is_named = node->rs->is_named;
+    addr32h = node->rs->addr;
+  } else if (node->ri) {
+    nickname = node->ri->nickname;
+    addr32h = node->ri->addr;
+  }
+
+  return format_node_description(buf,
+                                 node->identity,
+                                 is_named,
+                                 nickname,
+                                 NULL,
+                                 addr32h);
+}
+
+/** Return a human-readable description of the routerstatus_t <b>rs</b>.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+routerstatus_describe(const routerstatus_t *rs)
+{
+  static char buf[NODE_DESC_BUF_LEN];
+
+  if (!rs)
+    return "<null>";
+  return format_node_description(buf,
+                                 rs->identity_digest,
+                                 rs->is_named,
+                                 rs->nickname,
+                                 NULL,
+                                 rs->addr);
+}
+
+/** Return a human-readable description of the extend_info_t <b>ei</b>.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+extend_info_describe(const extend_info_t *ei)
+{
+  static char buf[NODE_DESC_BUF_LEN];
+
+  if (!ei)
+    return "<null>";
+  return format_node_description(buf,
+                                 ei->identity_digest,
+                                 0,
+                                 ei->nickname,
+                                 &ei->addr,
+                                 0);
+}
+
+/** Set <b>buf</b> (which must have MAX_VERBOSE_NICKNAME_LEN+1 bytes) to the
+ * verbose representation of the identity of <b>router</b>.  The format is:
+ *  A dollar sign.
+ *  The upper-case hexadecimal encoding of the SHA1 hash of router's identity.
+ *  A "=" if the router is named (no longer implemented); a "~" if it is not.
+ *  The router's nickname.
+ **/
+void
+router_get_verbose_nickname(char *buf, const routerinfo_t *router)
+{
+  buf[0] = '$';
+  base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest,
+                DIGEST_LEN);
+  buf[1+HEX_DIGEST_LEN] = '~';
+  strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1);
+}
+
+/** Forget that we have issued any router-related warnings, so that we'll
+ * warn again if we see the same errors. */
+void
+router_reset_warnings(void)
+{
+  if (warned_nonexistent_family) {
+    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
+    smartlist_clear(warned_nonexistent_family);
+  }
+}
+
+/** Given a router purpose, convert it to a string.  Don't call this on
+ * ROUTER_PURPOSE_UNKNOWN: The whole point of that value is that we don't
+ * know its string representation. */
+const char *
+router_purpose_to_string(uint8_t p)
+{
+  switch (p)
+    {
+    case ROUTER_PURPOSE_GENERAL: return "general";
+    case ROUTER_PURPOSE_BRIDGE: return "bridge";
+    case ROUTER_PURPOSE_CONTROLLER: return "controller";
+    default:
+      tor_assert(0);
+    }
+  return NULL;
+}
+
+/** Given a string, convert it to a router purpose. */
+uint8_t
+router_purpose_from_string(const char *s)
+{
+  if (!strcmp(s, "general"))
+    return ROUTER_PURPOSE_GENERAL;
+  else if (!strcmp(s, "bridge"))
+    return ROUTER_PURPOSE_BRIDGE;
+  else if (!strcmp(s, "controller"))
+    return ROUTER_PURPOSE_CONTROLLER;
+  else
+    return ROUTER_PURPOSE_UNKNOWN;
+}
+
+/** Release all static resources held in router.c */
+void
+router_free_all(void)
+{
+  crypto_pk_free(onionkey);
+  crypto_pk_free(lastonionkey);
+  crypto_pk_free(server_identitykey);
+  crypto_pk_free(client_identitykey);
+
+  tor_mutex_free(key_lock);
+  routerinfo_free(desc_routerinfo);
+  extrainfo_free(desc_extrainfo);
+  crypto_pk_free(authority_signing_key);
+  authority_cert_free(authority_key_certificate);
+  crypto_pk_free(legacy_signing_key);
+  authority_cert_free(legacy_key_certificate);
+
+  memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key));
+  memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
+
+  if (warned_nonexistent_family) {
+    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
+    smartlist_free(warned_nonexistent_family);
+  }
+}
+
+/** Return a smartlist of tor_addr_port_t's with all the OR ports of
+    <b>ri</b>. Note that freeing of the items in the list as well as
+    the smartlist itself is the callers responsibility. */
+smartlist_t *
+router_get_all_orports(const routerinfo_t *ri)
+{
+  tor_assert(ri);
+  node_t fake_node;
+  memset(&fake_node, 0, sizeof(fake_node));
+  /* we don't modify ri, fake_node is passed as a const node_t *
+   */
+  fake_node.ri = (routerinfo_t *)ri;
+  return node_get_all_orports(&fake_node);
+}
diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h
new file mode 100644
index 0000000000..51ac365798
--- /dev/null
+++ b/src/feature/relay/router.h
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file router.h
+ * \brief Header file for router.c.
+ **/
+
+#ifndef TOR_ROUTER_H
+#define TOR_ROUTER_H
+
+#include "lib/testsupport/testsupport.h"
+
+struct curve25519_keypair_t;
+struct ed25519_keypair_t;
+
+#define TOR_ROUTERINFO_ERROR_NO_EXT_ADDR     (-1)
+#define TOR_ROUTERINFO_ERROR_CANNOT_PARSE    (-2)
+#define TOR_ROUTERINFO_ERROR_NOT_A_SERVER    (-3)
+#define TOR_ROUTERINFO_ERROR_DIGEST_FAILED   (-4)
+#define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5)
+#define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6)
+
+crypto_pk_t *get_onion_key(void);
+time_t get_onion_key_set_at(void);
+void set_server_identity_key(crypto_pk_t *k);
+crypto_pk_t *get_server_identity_key(void);
+int server_identity_key_is_set(void);
+void set_client_identity_key(crypto_pk_t *k);
+crypto_pk_t *get_tlsclient_identity_key(void);
+int client_identity_key_is_set(void);
+MOCK_DECL(authority_cert_t *, get_my_v3_authority_cert, (void));
+crypto_pk_t *get_my_v3_authority_signing_key(void);
+authority_cert_t *get_my_v3_legacy_cert(void);
+crypto_pk_t *get_my_v3_legacy_signing_key(void);
+void dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last);
+void expire_old_onion_keys(void);
+void rotate_onion_key(void);
+crypto_pk_t *init_key_from_file(const char *fname, int generate,
+                                    int severity, int log_greeting);
+void v3_authority_check_key_expiry(void);
+int get_onion_key_lifetime(void);
+int get_onion_key_grace_period(void);
+
+di_digest256_map_t *construct_ntor_key_map(void);
+void ntor_key_map_free_(di_digest256_map_t *map);
+#define ntor_key_map_free(map) \
+  FREE_AND_NULL(di_digest256_map_t, ntor_key_map_free_, (map))
+
+int router_initialize_tls_context(void);
+int init_keys(void);
+int init_keys_client(void);
+
+int check_whether_orport_reachable(const or_options_t *options);
+int check_whether_dirport_reachable(const or_options_t *options);
+int dir_server_mode(const or_options_t *options);
+void router_do_reachability_checks(int test_or, int test_dir);
+void router_orport_found_reachable(void);
+void router_dirport_found_reachable(void);
+void router_perform_bandwidth_test(int num_circs, time_t now);
+
+int net_is_disabled(void);
+int net_is_completely_disabled(void);
+
+int authdir_mode(const or_options_t *options);
+int authdir_mode_handles_descs(const or_options_t *options, int purpose);
+int authdir_mode_publishes_statuses(const or_options_t *options);
+int authdir_mode_tests_reachability(const or_options_t *options);
+int authdir_mode_bridge(const or_options_t *options);
+
+uint16_t router_get_active_listener_port_by_type_af(int listener_type,
+                                                    sa_family_t family);
+uint16_t router_get_advertised_or_port(const or_options_t *options);
+uint16_t router_get_advertised_or_port_by_af(const or_options_t *options,
+                                             sa_family_t family);
+uint16_t router_get_advertised_dir_port(const or_options_t *options,
+                                        uint16_t dirport);
+
+MOCK_DECL(int, server_mode, (const or_options_t *options));
+MOCK_DECL(int, public_server_mode, (const or_options_t *options));
+MOCK_DECL(int, advertised_server_mode, (void));
+int proxy_mode(const or_options_t *options);
+void consider_publishable_server(int force);
+int should_refuse_unknown_exits(const or_options_t *options);
+
+void router_upload_dir_desc_to_dirservers(int force);
+void mark_my_descriptor_dirty_if_too_old(time_t now);
+void mark_my_descriptor_dirty(const char *reason);
+void check_descriptor_bandwidth_changed(time_t now);
+void check_descriptor_ipaddress_changed(time_t now);
+void router_new_address_suggestion(const char *suggestion,
+                                   const dir_connection_t *d_conn);
+int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port);
+MOCK_DECL(int, router_my_exit_policy_is_reject_star,(void));
+MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo_with_err,(int *err));
+extrainfo_t *router_get_my_extrainfo(void);
+const char *router_get_my_descriptor(void);
+const char *router_get_descriptor_gen_reason(void);
+int router_digest_is_me(const char *digest);
+const uint8_t *router_get_my_id_digest(void);
+int router_extrainfo_digest_is_me(const char *digest);
+int router_is_me(const routerinfo_t *router);
+MOCK_DECL(int,router_pick_published_address,(const or_options_t *options,
+                                             uint32_t *addr,
+                                             int cache_only));
+int router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e);
+int router_rebuild_descriptor(int force);
+char *router_dump_router_to_string(routerinfo_t *router,
+                             const crypto_pk_t *ident_key,
+                             const crypto_pk_t *tap_key,
+                             const struct curve25519_keypair_t *ntor_keypair,
+                             const struct ed25519_keypair_t *signing_keypair);
+char *router_dump_exit_policy_to_string(const routerinfo_t *router,
+                                         int include_ipv4,
+                                         int include_ipv6);
+void router_get_prim_orport(const routerinfo_t *router,
+                            tor_addr_port_t *addr_port_out);
+void router_get_pref_orport(const routerinfo_t *router,
+                            tor_addr_port_t *addr_port_out);
+void router_get_pref_ipv6_orport(const routerinfo_t *router,
+                                 tor_addr_port_t *addr_port_out);
+int router_ipv6_preferred(const routerinfo_t *router);
+int router_has_addr(const routerinfo_t *router, const tor_addr_t *addr);
+int router_has_orport(const routerinfo_t *router,
+                      const tor_addr_port_t *orport);
+int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo,
+                             crypto_pk_t *ident_key,
+                             const struct ed25519_keypair_t *signing_keypair);
+int is_legal_nickname(const char *s);
+int is_legal_nickname_or_hexdigest(const char *s);
+int is_legal_hexdigest(const char *s);
+
+const char *router_describe(const routerinfo_t *ri);
+const char *node_describe(const node_t *node);
+const char *routerstatus_describe(const routerstatus_t *ri);
+const char *extend_info_describe(const extend_info_t *ei);
+
+const char *routerinfo_err_to_string(int err);
+int routerinfo_err_is_transient(int err);
+
+void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
+void router_reset_warnings(void);
+void router_reset_reachability(void);
+void router_free_all(void);
+
+const char *router_purpose_to_string(uint8_t p);
+uint8_t router_purpose_from_string(const char *s);
+
+smartlist_t *router_get_all_orports(const routerinfo_t *ri);
+
+#ifdef ROUTER_PRIVATE
+/* Used only by router.c and test.c */
+STATIC void get_platform_str(char *platform, size_t len);
+STATIC int router_write_fingerprint(int hashed);
+#endif
+
+#endif /* !defined(TOR_ROUTER_H) */
diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c
new file mode 100644
index 0000000000..bb04a8b220
--- /dev/null
+++ b/src/feature/relay/routerkeys.c
@@ -0,0 +1,1413 @@
+/* Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerkeys.c
+ *
+ * \brief Functions and structures to handle generating and maintaining the
+ *  set of keypairs necessary to be an OR.
+ *
+ * The keys handled here now are the Ed25519 keys that Tor relays use to sign
+ * descriptors, authenticate themselves on links, and identify one another
+ * uniquely.  Other keys are maintained in router.c and rendservice.c.
+ *
+ * (TODO: The keys in router.c should go here too.)
+ */
+
+#include "or/or.h"
+#include "or/config.h"
+#include "or/router.h"
+#include "or/routerkeys.h"
+#include "or/torcert.h"
+
+#include "lib/crypt_ops/crypto_pwbox.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/term/getpass.h"
+#include "lib/tls/tortls.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+#define ENC_KEY_HEADER "Boxed Ed25519 key"
+#define ENC_KEY_TAG "master"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* DOCDOC */
+static ssize_t
+do_getpass(const char *prompt, char *buf, size_t buflen,
+           int twice, const or_options_t *options)
+{
+  if (options->keygen_force_passphrase == FORCE_PASSPHRASE_OFF) {
+    tor_assert(buflen);
+    buf[0] = 0;
+    return 0;
+  }
+
+  char *prompt2 = NULL;
+  char *buf2 = NULL;
+  int fd = -1;
+  ssize_t length = -1;
+
+  if (options->use_keygen_passphrase_fd) {
+    twice = 0;
+    fd = options->keygen_passphrase_fd;
+    length = read_all_from_fd(fd, buf, buflen-1);
+    if (length >= 0)
+      buf[length] = 0;
+    goto done_reading;
+  }
+
+  if (twice) {
+    const char msg[] = "One more time:";
+    size_t p2len = strlen(prompt) + 1;
+    if (p2len < sizeof(msg))
+      p2len = sizeof(msg);
+    prompt2 = tor_malloc(p2len);
+    memset(prompt2, ' ', p2len);
+    memcpy(prompt2 + p2len - sizeof(msg), msg, sizeof(msg));
+
+    buf2 = tor_malloc_zero(buflen);
+  }
+
+  while (1) {
+    length = tor_getpass(prompt, buf, buflen);
+    if (length < 0)
+      goto done_reading;
+
+    if (! twice)
+      break;
+
+    ssize_t length2 = tor_getpass(prompt2, buf2, buflen);
+
+    if (length != length2 || tor_memneq(buf, buf2, length)) {
+      fprintf(stderr, "That didn't match.\n");
+    } else {
+      break;
+    }
+  }
+
+ done_reading:
+  if (twice) {
+    tor_free(prompt2);
+    memwipe(buf2, 0, buflen);
+    tor_free(buf2);
+  }
+
+  if (options->keygen_force_passphrase == FORCE_PASSPHRASE_ON && length == 0)
+    return -1;
+
+  return length;
+}
+
+/* DOCDOC */
+int
+read_encrypted_secret_key(ed25519_secret_key_t *out,
+                          const char *fname)
+{
+  int r = -1;
+  uint8_t *secret = NULL;
+  size_t secret_len = 0;
+  char pwbuf[256];
+  uint8_t encrypted_key[256];
+  char *tag = NULL;
+  int saved_errno = 0;
+
+  ssize_t encrypted_len = crypto_read_tagged_contents_from_file(fname,
+                                          ENC_KEY_HEADER,
+                                          &tag,
+                                          encrypted_key,
+                                          sizeof(encrypted_key));
+  if (encrypted_len < 0) {
+    saved_errno = errno;
+    log_info(LD_OR, "%s is missing", fname);
+    r = 0;
+    goto done;
+  }
+  if (strcmp(tag, ENC_KEY_TAG)) {
+    saved_errno = EINVAL;
+    goto done;
+  }
+
+  while (1) {
+    ssize_t pwlen =
+      do_getpass("Enter passphrase for master key:", pwbuf, sizeof(pwbuf), 0,
+                 get_options());
+    if (pwlen < 0) {
+      saved_errno = EINVAL;
+      goto done;
+    }
+    const int r_unbox = crypto_unpwbox(&secret, &secret_len,
+                                       encrypted_key, encrypted_len,
+                                       pwbuf, pwlen);
+    if (r_unbox == UNPWBOX_CORRUPTED) {
+      log_err(LD_OR, "%s is corrupted.", fname);
+      saved_errno = EINVAL;
+      goto done;
+    } else if (r_unbox == UNPWBOX_OKAY) {
+      break;
+    }
+
+    /* Otherwise, passphrase is bad, so try again till user does ctrl-c or gets
+     * it right. */
+  }
+
+  if (secret_len != ED25519_SECKEY_LEN) {
+    log_err(LD_OR, "%s is corrupted.", fname);
+    saved_errno = EINVAL;
+    goto done;
+  }
+  memcpy(out->seckey, secret, ED25519_SECKEY_LEN);
+  r = 1;
+
+ done:
+  memwipe(encrypted_key, 0, sizeof(encrypted_key));
+  memwipe(pwbuf, 0, sizeof(pwbuf));
+  tor_free(tag);
+  if (secret) {
+    memwipe(secret, 0, secret_len);
+    tor_free(secret);
+  }
+  if (saved_errno)
+    errno = saved_errno;
+  return r;
+}
+
+/* DOCDOC */
+int
+write_encrypted_secret_key(const ed25519_secret_key_t *key,
+                           const char *fname)
+{
+  int r = -1;
+  char pwbuf0[256];
+  uint8_t *encrypted_key = NULL;
+  size_t encrypted_len = 0;
+
+  if (do_getpass("Enter new passphrase:", pwbuf0, sizeof(pwbuf0), 1,
+                 get_options()) < 0) {
+    log_warn(LD_OR, "NO/failed passphrase");
+    return -1;
+  }
+
+  if (strlen(pwbuf0) == 0) {
+    if (get_options()->keygen_force_passphrase == FORCE_PASSPHRASE_ON)
+      return -1;
+    else
+      return 0;
+  }
+
+  if (crypto_pwbox(&encrypted_key, &encrypted_len,
+                   key->seckey, sizeof(key->seckey),
+                   pwbuf0, strlen(pwbuf0),  0) < 0) {
+    log_warn(LD_OR, "crypto_pwbox failed!?");
+    goto done;
+  }
+  if (crypto_write_tagged_contents_to_file(fname,
+                                           ENC_KEY_HEADER,
+                                           ENC_KEY_TAG,
+                                           encrypted_key, encrypted_len) < 0)
+    goto done;
+  r = 1;
+ done:
+  if (encrypted_key) {
+    memwipe(encrypted_key, 0, encrypted_len);
+    tor_free(encrypted_key);
+  }
+  memwipe(pwbuf0, 0, sizeof(pwbuf0));
+  return r;
+}
+
+/* DOCDOC */
+static int
+write_secret_key(const ed25519_secret_key_t *key, int encrypted,
+                 const char *fname,
+                 const char *fname_tag,
+                 const char *encrypted_fname)
+{
+  if (encrypted) {
+    int r = write_encrypted_secret_key(key, encrypted_fname);
+    if (r == 1) {
+      /* Success! */
+
+      /* Try to unlink the unencrypted key, if any existed before */
+      if (strcmp(fname, encrypted_fname))
+        unlink(fname);
+      return r;
+    } else if (r != 0) {
+      /* Unrecoverable failure! */
+      return r;
+    }
+
+    fprintf(stderr, "Not encrypting the secret key.\n");
+  }
+  return ed25519_seckey_write_to_file(key, fname, fname_tag);
+}
+
+/**
+ * Read an ed25519 key and associated certificates from files beginning with
+ * <b>fname</b>, with certificate type <b>cert_type</b>.  On failure, return
+ * NULL; on success return the keypair.
+ *
+ * If INIT_ED_KEY_CREATE is set in <b>flags</b>, then create the key (and
+ * certificate if requested) if it doesn't exist, and save it to disk.
+ *
+ * If INIT_ED_KEY_NEEDCERT is set in <b>flags</b>, load/create a certificate
+ * too and store it in *<b>cert_out</b>.  Fail if the cert can't be
+ * found/created.  To create a certificate, <b>signing_key</b> must be set to
+ * the key that should sign it; <b>now</b> to the current time, and
+ * <b>lifetime</b> to the lifetime of the key.
+ *
+ * If INIT_ED_KEY_REPLACE is set in <b>flags</b>, then create and save new key
+ * whether we can read the old one or not.
+ *
+ * If INIT_ED_KEY_EXTRA_STRONG is set in <b>flags</b>, set the extra_strong
+ * flag when creating the secret key.
+ *
+ * If INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT is set in <b>flags</b>, and
+ * we create a new certificate, create it with the signing key embedded.
+ *
+ * If INIT_ED_KEY_SPLIT is set in <b>flags</b>, and we create a new key,
+ * store the public key in a separate file from the secret key.
+ *
+ * If INIT_ED_KEY_MISSING_SECRET_OK is set in <b>flags</b>, and we find a
+ * public key file but no secret key file, return successfully anyway.
+ *
+ * If INIT_ED_KEY_OMIT_SECRET is set in <b>flags</b>, do not try to load a
+ * secret key unless no public key is found.  Do not return a secret key. (but
+ * create and save one if needed).
+ *
+ * If INIT_ED_KEY_NO_LOAD_SECRET is set in <b>flags</b>, don't try to load
+ * a secret key, no matter what.
+ *
+ * If INIT_ED_KEY_TRY_ENCRYPTED is set, we look for an encrypted secret key
+ * and consider encrypting any new secret key.
+ *
+ * If INIT_ED_KEY_NO_REPAIR is set, and there is any issue loading the keys
+ * from disk _other than their absence_ (full or partial), we do not try to
+ * replace them.
+ *
+ * If INIT_ED_KEY_SUGGEST_KEYGEN is set, have log messages about failures
+ * refer to the --keygen option.
+ *
+ * If INIT_ED_KEY_EXPLICIT_FNAME is set, use the provided file name for the
+ * secret key file, encrypted or not.
+ */
+ed25519_keypair_t *
+ed_key_init_from_file(const char *fname, uint32_t flags,
+                      int severity,
+                      const ed25519_keypair_t *signing_key,
+                      time_t now,
+                      time_t lifetime,
+                      uint8_t cert_type,
+                      struct tor_cert_st **cert_out)
+{
+  char *secret_fname = NULL;
+  char *encrypted_secret_fname = NULL;
+  char *public_fname = NULL;
+  char *cert_fname = NULL;
+  const char *loaded_secret_fname = NULL;
+  int created_pk = 0, created_sk = 0, created_cert = 0;
+  const int try_to_load = ! (flags & INIT_ED_KEY_REPLACE);
+  const int encrypt_key = !! (flags & INIT_ED_KEY_TRY_ENCRYPTED);
+  const int norepair = !! (flags & INIT_ED_KEY_NO_REPAIR);
+  const int split = !! (flags & INIT_ED_KEY_SPLIT);
+  const int omit_secret = !! (flags & INIT_ED_KEY_OMIT_SECRET);
+  const int offline_secret = !! (flags & INIT_ED_KEY_OFFLINE_SECRET);
+  const int explicit_fname = !! (flags & INIT_ED_KEY_EXPLICIT_FNAME);
+
+  /* we don't support setting both of these flags at once. */
+  tor_assert((flags & (INIT_ED_KEY_NO_REPAIR|INIT_ED_KEY_NEEDCERT)) !=
+                      (INIT_ED_KEY_NO_REPAIR|INIT_ED_KEY_NEEDCERT));
+
+  char tag[8];
+  tor_snprintf(tag, sizeof(tag), "type%d", (int)cert_type);
+
+  tor_cert_t *cert = NULL;
+  char *got_tag = NULL;
+  ed25519_keypair_t *keypair = tor_malloc_zero(sizeof(ed25519_keypair_t));
+
+  if (explicit_fname) {
+    secret_fname = tor_strdup(fname);
+    encrypted_secret_fname = tor_strdup(fname);
+  } else {
+    tor_asprintf(&secret_fname, "%s_secret_key", fname);
+    tor_asprintf(&encrypted_secret_fname, "%s_secret_key_encrypted", fname);
+  }
+  tor_asprintf(&public_fname, "%s_public_key", fname);
+  tor_asprintf(&cert_fname, "%s_cert", fname);
+
+  /* Try to read the secret key. */
+  int have_secret = 0;
+  int load_secret = try_to_load &&
+    !offline_secret &&
+    (!omit_secret || file_status(public_fname)==FN_NOENT);
+  if (load_secret) {
+    int rv = ed25519_seckey_read_from_file(&keypair->seckey,
+                                           &got_tag, secret_fname);
+    if (rv == 0) {
+      have_secret = 1;
+      loaded_secret_fname = secret_fname;
+      tor_assert(got_tag);
+    } else {
+      if (errno != ENOENT && norepair) {
+        tor_log(severity, LD_OR, "Unable to read %s: %s", secret_fname,
+                strerror(errno));
+        goto err;
+      }
+    }
+  }
+
+  /* Should we try for an encrypted key? */
+  int have_encrypted_secret_file = 0;
+  if (!have_secret && try_to_load && encrypt_key) {
+    int r = read_encrypted_secret_key(&keypair->seckey,
+                                      encrypted_secret_fname);
+    if (r > 0) {
+      have_secret = 1;
+      have_encrypted_secret_file = 1;
+      tor_free(got_tag); /* convince coverity we aren't leaking */
+      got_tag = tor_strdup(tag);
+      loaded_secret_fname = encrypted_secret_fname;
+    } else if (errno != ENOENT && norepair) {
+      tor_log(severity, LD_OR, "Unable to read %s: %s",
+              encrypted_secret_fname, strerror(errno));
+      goto err;
+    }
+  } else {
+    if (try_to_load) {
+      /* Check if it's there anyway, so we don't replace it. */
+      if (file_status(encrypted_secret_fname) != FN_NOENT)
+        have_encrypted_secret_file = 1;
+    }
+  }
+
+  if (have_secret) {
+    if (strcmp(got_tag, tag)) {
+      tor_log(severity, LD_OR, "%s has wrong tag", loaded_secret_fname);
+      goto err;
+    }
+    /* Derive the public key */
+    if (ed25519_public_key_generate(&keypair->pubkey, &keypair->seckey)<0) {
+      tor_log(severity, LD_OR, "%s can't produce a public key",
+              loaded_secret_fname);
+      goto err;
+    }
+  }
+
+  /* If we do split keys here, try to read the pubkey. */
+  int found_public = 0;
+  if (try_to_load && (!have_secret || split)) {
+    ed25519_public_key_t pubkey_tmp;
+    tor_free(got_tag);
+    found_public = ed25519_pubkey_read_from_file(&pubkey_tmp,
+                                                 &got_tag, public_fname) == 0;
+    if (!found_public && errno != ENOENT && norepair) {
+      tor_log(severity, LD_OR, "Unable to read %s: %s", public_fname,
+              strerror(errno));
+      goto err;
+    }
+    if (found_public && strcmp(got_tag, tag)) {
+      tor_log(severity, LD_OR, "%s has wrong tag", public_fname);
+      goto err;
+    }
+    if (found_public) {
+      if (have_secret) {
+        /* If we have a secret key and we're reloading the public key,
+         * the key must match! */
+        if (! ed25519_pubkey_eq(&keypair->pubkey, &pubkey_tmp)) {
+          tor_log(severity, LD_OR, "%s does not match %s!  If you are trying "
+                  "to restore from backup, make sure you didn't mix up the "
+                  "key files. If you are absolutely sure that %s is the right "
+                  "key for this relay, delete %s or move it out of the way.",
+                  public_fname, loaded_secret_fname,
+                  loaded_secret_fname, public_fname);
+          goto err;
+        }
+      } else {
+        /* We only have the public key; better use that. */
+        tor_assert(split);
+        memcpy(&keypair->pubkey, &pubkey_tmp, sizeof(pubkey_tmp));
+      }
+    } else {
+      /* We have no public key file, but we do have a secret key, make the
+       * public key file! */
+      if (have_secret) {
+        if (ed25519_pubkey_write_to_file(&keypair->pubkey, public_fname, tag)
+            < 0) {
+          tor_log(severity, LD_OR, "Couldn't repair %s", public_fname);
+          goto err;
+        } else {
+          tor_log(LOG_NOTICE, LD_OR,
+                  "Found secret key but not %s. Regenerating.",
+                  public_fname);
+        }
+      }
+    }
+  }
+
+  /* If the secret key is absent and it's not allowed to be, fail. */
+  if (!have_secret && found_public &&
+      !(flags & INIT_ED_KEY_MISSING_SECRET_OK)) {
+    if (have_encrypted_secret_file) {
+      tor_log(severity, LD_OR, "We needed to load a secret key from %s, "
+              "but it was encrypted. Try 'tor --keygen' instead, so you "
+              "can enter the passphrase.",
+              secret_fname);
+    } else if (offline_secret) {
+      tor_log(severity, LD_OR, "We wanted to load a secret key from %s, "
+              "but you're keeping it offline. (OfflineMasterKey is set.)",
+              secret_fname);
+    } else {
+      tor_log(severity, LD_OR, "We needed to load a secret key from %s, "
+              "but couldn't find it. %s", secret_fname,
+              (flags & INIT_ED_KEY_SUGGEST_KEYGEN) ?
+              "If you're keeping your master secret key offline, you will "
+              "need to run 'tor --keygen' to generate new signing keys." :
+              "Did you forget to copy it over when you copied the rest of the "
+              "signing key material?");
+    }
+    goto err;
+  }
+
+  /* If it's absent, and we're not supposed to make a new keypair, fail. */
+  if (!have_secret && !found_public && !(flags & INIT_ED_KEY_CREATE)) {
+    if (split) {
+      tor_log(severity, LD_OR, "No key found in %s or %s.",
+              secret_fname, public_fname);
+    } else {
+      tor_log(severity, LD_OR, "No key found in %s.", secret_fname);
+    }
+    goto err;
+  }
+
+  /* If the secret key is absent, but the encrypted key would be present,
+   * that's an error */
+  if (!have_secret && !found_public && have_encrypted_secret_file) {
+    tor_assert(!encrypt_key);
+    tor_log(severity, LD_OR, "Found an encrypted secret key, "
+            "but not public key file %s!", public_fname);
+    goto err;
+  }
+
+  /* if it's absent, make a new keypair... */
+  if (!have_secret && !found_public) {
+    tor_free(keypair);
+    keypair = ed_key_new(signing_key, flags, now, lifetime,
+                         cert_type, &cert);
+    if (!keypair) {
+      tor_log(severity, LD_OR, "Couldn't create keypair");
+      goto err;
+    }
+    created_pk = created_sk = created_cert = 1;
+  }
+
+  /* Write it to disk if we're supposed to do with a new passphrase, or if
+   * we just created it. */
+  if (created_sk || (have_secret && get_options()->change_key_passphrase)) {
+    if (write_secret_key(&keypair->seckey,
+                         encrypt_key,
+                         secret_fname, tag, encrypted_secret_fname) < 0
+        ||
+        (split &&
+         ed25519_pubkey_write_to_file(&keypair->pubkey, public_fname, tag) < 0)
+        ||
+        (cert &&
+         crypto_write_tagged_contents_to_file(cert_fname, "ed25519v1-cert",
+                                 tag, cert->encoded, cert->encoded_len) < 0)) {
+      tor_log(severity, LD_OR, "Couldn't write keys or cert to file.");
+      goto err;
+    }
+    goto done;
+  }
+
+  /* If we're not supposed to get a cert, we're done. */
+  if (! (flags & INIT_ED_KEY_NEEDCERT))
+    goto done;
+
+  /* Read a cert. */
+  tor_free(got_tag);
+  uint8_t certbuf[256];
+  ssize_t cert_body_len = crypto_read_tagged_contents_from_file(
+                 cert_fname, "ed25519v1-cert",
+                 &got_tag, certbuf, sizeof(certbuf));
+  if (cert_body_len >= 0 && !strcmp(got_tag, tag))
+    cert = tor_cert_parse(certbuf, cert_body_len);
+
+  /* If we got it, check it to the extent we can. */
+  int bad_cert = 0;
+
+  if (! cert) {
+    tor_log(severity, LD_OR, "Cert was unparseable");
+    bad_cert = 1;
+  } else if (!tor_memeq(cert->signed_key.pubkey, keypair->pubkey.pubkey,
+                        ED25519_PUBKEY_LEN)) {
+    tor_log(severity, LD_OR, "Cert was for wrong key");
+    bad_cert = 1;
+  } else if (signing_key &&
+             tor_cert_checksig(cert, &signing_key->pubkey, now) < 0) {
+    tor_log(severity, LD_OR, "Can't check certificate: %s",
+            tor_cert_describe_signature_status(cert));
+    bad_cert = 1;
+  } else if (cert->cert_expired) {
+    tor_log(severity, LD_OR, "Certificate is expired");
+    bad_cert = 1;
+  } else if (signing_key && cert->signing_key_included &&
+             ! ed25519_pubkey_eq(&signing_key->pubkey, &cert->signing_key)) {
+    tor_log(severity, LD_OR, "Certificate signed by unexpectd key!");
+    bad_cert = 1;
+  }
+
+  if (bad_cert) {
+    tor_cert_free(cert);
+    cert = NULL;
+  }
+
+  /* If we got a cert, we're done. */
+  if (cert)
+    goto done;
+
+  /* If we didn't get a cert, and we're not supposed to make one, fail. */
+  if (!signing_key || !(flags & INIT_ED_KEY_CREATE)) {
+    tor_log(severity, LD_OR, "Without signing key, can't create certificate");
+    goto err;
+  }
+
+  /* We have keys but not a certificate, so make one. */
+  uint32_t cert_flags = 0;
+  if (flags & INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT)
+    cert_flags |= CERT_FLAG_INCLUDE_SIGNING_KEY;
+  cert = tor_cert_create(signing_key, cert_type,
+                         &keypair->pubkey,
+                         now, lifetime,
+                         cert_flags);
+
+  if (! cert) {
+    tor_log(severity, LD_OR, "Couldn't create certificate");
+    goto err;
+  }
+
+  /* Write it to disk. */
+  created_cert = 1;
+  if (crypto_write_tagged_contents_to_file(cert_fname, "ed25519v1-cert",
+                             tag, cert->encoded, cert->encoded_len) < 0) {
+    tor_log(severity, LD_OR, "Couldn't write cert to disk.");
+    goto err;
+  }
+
+ done:
+  if (cert_out)
+    *cert_out = cert;
+  else
+    tor_cert_free(cert);
+
+  goto cleanup;
+
+ err:
+  if (keypair)
+    memwipe(keypair, 0, sizeof(*keypair));
+  tor_free(keypair);
+  tor_cert_free(cert);
+  if (cert_out)
+    *cert_out = NULL;
+  if (created_sk)
+    unlink(secret_fname);
+  if (created_pk)
+    unlink(public_fname);
+  if (created_cert)
+    unlink(cert_fname);
+
+ cleanup:
+  tor_free(encrypted_secret_fname);
+  tor_free(secret_fname);
+  tor_free(public_fname);
+  tor_free(cert_fname);
+  tor_free(got_tag);
+
+  return keypair;
+}
+
+/**
+ * Create a new signing key and (optionally) certficiate; do not read or write
+ * from disk.  See ed_key_init_from_file() for more information.
+ */
+ed25519_keypair_t *
+ed_key_new(const ed25519_keypair_t *signing_key,
+           uint32_t flags,
+           time_t now,
+           time_t lifetime,
+           uint8_t cert_type,
+           struct tor_cert_st **cert_out)
+{
+  if (cert_out)
+    *cert_out = NULL;
+
+  const int extra_strong = !! (flags & INIT_ED_KEY_EXTRA_STRONG);
+  ed25519_keypair_t *keypair = tor_malloc_zero(sizeof(ed25519_keypair_t));
+  if (ed25519_keypair_generate(keypair, extra_strong) < 0)
+    goto err;
+
+  if (! (flags & INIT_ED_KEY_NEEDCERT))
+    return keypair;
+
+  tor_assert(signing_key);
+  tor_assert(cert_out);
+  uint32_t cert_flags = 0;
+  if (flags & INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT)
+    cert_flags |= CERT_FLAG_INCLUDE_SIGNING_KEY;
+  tor_cert_t *cert = tor_cert_create(signing_key, cert_type,
+                                     &keypair->pubkey,
+                                     now, lifetime,
+                                     cert_flags);
+  if (! cert)
+    goto err;
+
+  *cert_out = cert;
+  return keypair;
+
+ err:
+  tor_free(keypair);
+  return NULL;
+}
+
+static ed25519_keypair_t *master_identity_key = NULL;
+static ed25519_keypair_t *master_signing_key = NULL;
+static ed25519_keypair_t *current_auth_key = NULL;
+static tor_cert_t *signing_key_cert = NULL;
+static tor_cert_t *link_cert_cert = NULL;
+static tor_cert_t *auth_key_cert = NULL;
+
+static uint8_t *rsa_ed_crosscert = NULL;
+static size_t rsa_ed_crosscert_len = 0;
+static time_t rsa_ed_crosscert_expiration = 0;
+
+/**
+ * Running as a server: load, reload, or refresh our ed25519 keys and
+ * certificates, creating and saving new ones as needed.
+ *
+ * Return -1 on failure; 0 on success if the signing key was not replaced;
+ * and 1 on success if the signing key was replaced.
+ */
+int
+load_ed_keys(const or_options_t *options, time_t now)
+{
+  ed25519_keypair_t *id = NULL;
+  ed25519_keypair_t *sign = NULL;
+  ed25519_keypair_t *auth = NULL;
+  const ed25519_keypair_t *sign_signing_key_with_id = NULL;
+  const ed25519_keypair_t *use_signing = NULL;
+  const tor_cert_t *check_signing_cert = NULL;
+  tor_cert_t *sign_cert = NULL;
+  tor_cert_t *auth_cert = NULL;
+  int signing_key_changed = 0;
+
+  // It is later than 1972, since otherwise there would be no C compilers.
+  // (Try to diagnose #22466.)
+  tor_assert_nonfatal(now >= 2 * 365 * 86400);
+
+#define FAIL(msg) do {                          \
+    log_warn(LD_OR, (msg));                     \
+    goto err;                                   \
+  } while (0)
+#define SET_KEY(key, newval) do {               \
+    if ((key) != (newval))                      \
+      ed25519_keypair_free(key);                \
+    key = (newval);                             \
+  } while (0)
+#define SET_CERT(cert, newval) do {             \
+    if ((cert) != (newval))                     \
+      tor_cert_free(cert);                      \
+    cert = (newval);                            \
+  } while (0)
+#define HAPPENS_SOON(when, interval)            \
+  ((when) < now + (interval))
+#define EXPIRES_SOON(cert, interval)            \
+  (!(cert) || HAPPENS_SOON((cert)->valid_until, (interval)))
+
+  /* XXXX support encrypted identity keys fully */
+
+  /* First try to get the signing key to see how it is. */
+  {
+    char *fname =
+      options_get_keydir_fname(options, "ed25519_signing");
+    sign = ed_key_init_from_file(
+               fname,
+               INIT_ED_KEY_NEEDCERT|
+               INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT,
+               LOG_INFO,
+               NULL, 0, 0, CERT_TYPE_ID_SIGNING, &sign_cert);
+    tor_free(fname);
+    check_signing_cert = sign_cert;
+    use_signing = sign;
+  }
+
+  if (use_signing) {
+    /* We loaded a signing key with its certificate.  */
+    if (! master_signing_key) {
+      /* We didn't know one before! */
+      signing_key_changed = 1;
+    } else if (! ed25519_pubkey_eq(&use_signing->pubkey,
+                                   &master_signing_key->pubkey) ||
+               ! tor_memeq(use_signing->seckey.seckey,
+                           master_signing_key->seckey.seckey,
+                           ED25519_SECKEY_LEN)) {
+      /* We loaded a different signing key than the one we knew before. */
+      signing_key_changed = 1;
+    }
+  }
+
+  if (!use_signing && master_signing_key) {
+    /* We couldn't load a signing key, but we already had one loaded */
+    check_signing_cert = signing_key_cert;
+    use_signing = master_signing_key;
+  }
+
+  const int offline_master =
+    options->OfflineMasterKey && options->command != CMD_KEYGEN;
+  const int need_new_signing_key =
+    NULL == use_signing ||
+    EXPIRES_SOON(check_signing_cert, 0) ||
+    (options->command == CMD_KEYGEN && ! options->change_key_passphrase);
+  const int want_new_signing_key =
+    need_new_signing_key ||
+    EXPIRES_SOON(check_signing_cert, options->TestingSigningKeySlop);
+
+  /* We can only create a master key if we haven't been told that the
+   * master key will always be offline.  Also, if we have a signing key,
+   * then we shouldn't make a new master ID key. */
+  const int can_make_master_id_key = !offline_master &&
+    NULL == use_signing;
+
+  if (need_new_signing_key) {
+    log_notice(LD_OR, "It looks like I need to generate and sign a new "
+               "medium-term signing key, because %s. To do that, I "
+               "need to load%s the permanent master identity key. "
+               "If the master identity key was not moved or encrypted "
+               "with a passphrase, this will be done automatically and "
+               "no further action is required. Otherwise, provide the "
+               "necessary data using 'tor --keygen' to do it manually.",
+            (NULL == use_signing) ? "I don't have one" :
+            EXPIRES_SOON(check_signing_cert, 0) ? "the one I have is expired" :
+               "you asked me to make one with --keygen",
+            can_make_master_id_key ? " (or create)" : "");
+  } else if (want_new_signing_key && !offline_master) {
+    log_notice(LD_OR, "It looks like I should try to generate and sign a "
+               "new medium-term signing key, because the one I have is "
+               "going to expire soon. To do that, I'm going to have to "
+               "try to load the permanent master identity key. "
+               "If the master identity key was not moved or encrypted "
+               "with a passphrase, this will be done automatically and "
+               "no further action is required. Otherwise, provide the "
+               "necessary data using 'tor --keygen' to do it manually.");
+  } else if (want_new_signing_key) {
+    log_notice(LD_OR, "It looks like I should try to generate and sign a "
+               "new medium-term signing key, because the one I have is "
+               "going to expire soon. But OfflineMasterKey is set, so I "
+               "won't try to load a permanent master identity key. You "
+               "will need to use 'tor --keygen' to make a new signing "
+               "key and certificate.");
+  }
+
+  {
+    uint32_t flags =
+      (INIT_ED_KEY_SPLIT|
+       INIT_ED_KEY_EXTRA_STRONG|INIT_ED_KEY_NO_REPAIR);
+    if (can_make_master_id_key)
+      flags |= INIT_ED_KEY_CREATE;
+    if (! need_new_signing_key)
+      flags |= INIT_ED_KEY_MISSING_SECRET_OK;
+    if (! want_new_signing_key || offline_master)
+      flags |= INIT_ED_KEY_OMIT_SECRET;
+    if (offline_master)
+      flags |= INIT_ED_KEY_OFFLINE_SECRET;
+    if (options->command == CMD_KEYGEN)
+      flags |= INIT_ED_KEY_TRY_ENCRYPTED;
+
+    /* Check/Create the key directory */
+    if (create_keys_directory(options) < 0)
+      return -1;
+
+    char *fname;
+    if (options->master_key_fname) {
+      fname = tor_strdup(options->master_key_fname);
+      flags |= INIT_ED_KEY_EXPLICIT_FNAME;
+    } else {
+      fname = options_get_keydir_fname(options, "ed25519_master_id");
+    }
+    id = ed_key_init_from_file(
+             fname,
+             flags,
+             LOG_WARN, NULL, 0, 0, 0, NULL);
+    tor_free(fname);
+    if (!id) {
+      if (need_new_signing_key) {
+        if (offline_master)
+          FAIL("Can't load master identity key; OfflineMasterKey is set.");
+        else
+          FAIL("Missing identity key");
+      } else {
+        log_warn(LD_OR, "Master public key was absent; inferring from "
+                 "public key in signing certificate and saving to disk.");
+        tor_assert(check_signing_cert);
+        id = tor_malloc_zero(sizeof(*id));
+        memcpy(&id->pubkey, &check_signing_cert->signing_key,
+               sizeof(ed25519_public_key_t));
+        fname = options_get_keydir_fname(options,
+                                         "ed25519_master_id_public_key");
+        if (ed25519_pubkey_write_to_file(&id->pubkey, fname, "type0") < 0) {
+          log_warn(LD_OR, "Error while attempting to write master public key "
+                   "to disk");
+          tor_free(fname);
+          goto err;
+        }
+        tor_free(fname);
+      }
+    }
+    if (tor_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
+      sign_signing_key_with_id = NULL;
+    else
+      sign_signing_key_with_id = id;
+  }
+
+  if (master_identity_key &&
+      !ed25519_pubkey_eq(&id->pubkey, &master_identity_key->pubkey)) {
+    FAIL("Identity key on disk does not match key we loaded earlier!");
+  }
+
+  if (need_new_signing_key && NULL == sign_signing_key_with_id)
+    FAIL("Can't load master key make a new signing key.");
+
+  if (sign_cert) {
+    if (! sign_cert->signing_key_included)
+      FAIL("Loaded a signing cert with no key included!");
+    if (! ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey))
+      FAIL("The signing cert we have was not signed with the master key "
+           "we loaded!");
+    if (tor_cert_checksig(sign_cert, &id->pubkey, 0) < 0) {
+      log_warn(LD_OR, "The signing cert we loaded was not signed "
+               "correctly: %s!",
+               tor_cert_describe_signature_status(sign_cert));
+      goto err;
+    }
+  }
+
+  if (want_new_signing_key && sign_signing_key_with_id) {
+    uint32_t flags = (INIT_ED_KEY_CREATE|
+                      INIT_ED_KEY_REPLACE|
+                      INIT_ED_KEY_EXTRA_STRONG|
+                      INIT_ED_KEY_NEEDCERT|
+                      INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT);
+    char *fname =
+      options_get_keydir_fname(options, "ed25519_signing");
+    ed25519_keypair_free(sign);
+    tor_cert_free(sign_cert);
+    sign = ed_key_init_from_file(fname,
+                                 flags, LOG_WARN,
+                                 sign_signing_key_with_id, now,
+                                 options->SigningKeyLifetime,
+                                 CERT_TYPE_ID_SIGNING, &sign_cert);
+    tor_free(fname);
+    if (!sign)
+      FAIL("Missing signing key");
+    use_signing = sign;
+    signing_key_changed = 1;
+
+    tor_assert(sign_cert->signing_key_included);
+    tor_assert(ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey));
+    tor_assert(ed25519_pubkey_eq(&sign_cert->signed_key, &sign->pubkey));
+  } else if (want_new_signing_key) {
+    static ratelim_t missing_master = RATELIM_INIT(3600);
+    log_fn_ratelim(&missing_master, LOG_WARN, LD_OR,
+                   "Signing key will expire soon, but I can't load the "
+                   "master key to sign a new one!");
+  }
+
+  tor_assert(use_signing);
+
+  /* At this point we no longer need our secret identity key.  So wipe
+   * it, if we loaded it in the first place. */
+  memwipe(id->seckey.seckey, 0, sizeof(id->seckey));
+
+  if (options->command == CMD_KEYGEN)
+    goto end;
+
+  if (server_mode(options) &&
+      (!rsa_ed_crosscert ||
+       HAPPENS_SOON(rsa_ed_crosscert_expiration, 30*86400))) {
+    uint8_t *crosscert;
+    time_t expiration = now+6*30*86400; /* 6 months in the future. */
+    ssize_t crosscert_len = tor_make_rsa_ed25519_crosscert(&id->pubkey,
+                                                   get_server_identity_key(),
+                                                   expiration,
+                                                   &crosscert);
+    tor_free(rsa_ed_crosscert);
+    rsa_ed_crosscert_len = crosscert_len;
+    rsa_ed_crosscert = crosscert;
+    rsa_ed_crosscert_expiration = expiration;
+  }
+
+  if (!current_auth_key ||
+      signing_key_changed ||
+      EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop)) {
+    auth = ed_key_new(use_signing, INIT_ED_KEY_NEEDCERT,
+                      now,
+                      options->TestingAuthKeyLifetime,
+                      CERT_TYPE_SIGNING_AUTH, &auth_cert);
+
+    if (!auth)
+      FAIL("Can't create auth key");
+  }
+
+  /* We've generated or loaded everything.  Put them in memory. */
+
+ end:
+  if (! master_identity_key) {
+    SET_KEY(master_identity_key, id);
+  } else {
+    tor_free(id);
+  }
+  if (sign) {
+    SET_KEY(master_signing_key, sign);
+    SET_CERT(signing_key_cert, sign_cert);
+  }
+  if (auth) {
+    SET_KEY(current_auth_key, auth);
+    SET_CERT(auth_key_cert, auth_cert);
+  }
+
+  return signing_key_changed;
+ err:
+  ed25519_keypair_free(id);
+  ed25519_keypair_free(sign);
+  ed25519_keypair_free(auth);
+  tor_cert_free(sign_cert);
+  tor_cert_free(auth_cert);
+  return -1;
+}
+
+/**
+ * Retrieve our currently-in-use Ed25519 link certificate and id certificate,
+ * and, if they would expire soon (based on the time <b>now</b>, generate new
+ * certificates (without embedding the public part of the signing key inside).
+ * If <b>force</b> is true, always generate a new certificate.
+ *
+ * The signed_key from the current id->signing certificate will be used to
+ * sign the new key within newly generated X509 certificate.
+ *
+ * Returns -1 upon error.  Otherwise, returns 0 upon success (either when the
+ * current certificate is still valid, or when a new certificate was
+ * successfully generated, or no certificate was needed).
+ */
+int
+generate_ed_link_cert(const or_options_t *options, time_t now,
+                      int force)
+{
+  const tor_x509_cert_t *link_ = NULL, *id = NULL;
+  tor_cert_t *link_cert = NULL;
+
+  if (tor_tls_get_my_certs(1, &link_, &id) < 0 || link_ == NULL) {
+    if (!server_mode(options)) {
+        /* No need to make an Ed25519->Link cert: we are a client */
+      return 0;
+    }
+    log_warn(LD_OR, "Can't get my x509 link cert.");
+    return -1;
+  }
+
+  const common_digests_t *digests = tor_x509_cert_get_cert_digests(link_);
+
+  if (force == 0 &&
+      link_cert_cert &&
+      ! EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop) &&
+      fast_memeq(digests->d[DIGEST_SHA256], link_cert_cert->signed_key.pubkey,
+                 DIGEST256_LEN)) {
+    return 0;
+  }
+
+  ed25519_public_key_t dummy_key;
+  memcpy(dummy_key.pubkey, digests->d[DIGEST_SHA256], DIGEST256_LEN);
+
+  link_cert = tor_cert_create(get_master_signing_keypair(),
+                              CERT_TYPE_SIGNING_LINK,
+                              &dummy_key,
+                              now,
+                              options->TestingLinkCertLifetime, 0);
+
+  if (link_cert) {
+    SET_CERT(link_cert_cert, link_cert);
+  }
+  return 0;
+}
+
+#undef FAIL
+#undef SET_KEY
+#undef SET_CERT
+
+/**
+ * Return 1 if any of the following are true:
+ *
+ *   - if one of our Ed25519 signing, auth, or link certificates would expire
+ *     soon w.r.t. the time <b>now</b>,
+ *   - if we do not currently have a link certificate, or
+ *   - if our cached Ed25519 link certificate is not same as the one we're
+ *     currently using.
+ *
+ * Otherwise, returns 0.
+ */
+int
+should_make_new_ed_keys(const or_options_t *options, const time_t now)
+{
+  if (!master_identity_key ||
+      !master_signing_key ||
+      !current_auth_key ||
+      !link_cert_cert ||
+      EXPIRES_SOON(signing_key_cert, options->TestingSigningKeySlop) ||
+      EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop) ||
+      EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop))
+    return 1;
+
+  const tor_x509_cert_t *link_ = NULL, *id = NULL;
+
+  if (tor_tls_get_my_certs(1, &link_, &id) < 0 || link_ == NULL)
+    return 1;
+
+  const common_digests_t *digests = tor_x509_cert_get_cert_digests(link_);
+
+  if (!fast_memeq(digests->d[DIGEST_SHA256],
+                  link_cert_cert->signed_key.pubkey,
+                  DIGEST256_LEN)) {
+    return 1;
+  }
+
+  return 0;
+}
+
+#undef EXPIRES_SOON
+#undef HAPPENS_SOON
+
+#ifdef TOR_UNIT_TESTS
+/* Helper for unit tests: populate the ed25519 keys without saving or
+ * loading */
+void
+init_mock_ed_keys(const crypto_pk_t *rsa_identity_key)
+{
+  routerkeys_free_all();
+
+#define MAKEKEY(k)                                      \
+  k = tor_malloc_zero(sizeof(*k));                      \
+  if (ed25519_keypair_generate(k, 0) < 0) {             \
+    log_warn(LD_BUG, "Couldn't make a keypair");        \
+    goto err;                                           \
+  }
+  MAKEKEY(master_identity_key);
+  MAKEKEY(master_signing_key);
+  MAKEKEY(current_auth_key);
+#define MAKECERT(cert, signing, signed_, type, flags)            \
+  cert = tor_cert_create(signing,                                \
+                         type,                                   \
+                         &signed_->pubkey,                       \
+                         time(NULL), 86400,                      \
+                         flags);                                 \
+  if (!cert) {                                                   \
+    log_warn(LD_BUG, "Couldn't make a %s certificate!", #cert);  \
+    goto err;                                                    \
+  }
+
+  MAKECERT(signing_key_cert,
+           master_identity_key, master_signing_key, CERT_TYPE_ID_SIGNING,
+           CERT_FLAG_INCLUDE_SIGNING_KEY);
+  MAKECERT(auth_key_cert,
+           master_signing_key, current_auth_key, CERT_TYPE_SIGNING_AUTH, 0);
+
+  if (generate_ed_link_cert(get_options(), time(NULL), 0) < 0) {
+    log_warn(LD_BUG, "Couldn't make link certificate");
+    goto err;
+  }
+
+  rsa_ed_crosscert_len = tor_make_rsa_ed25519_crosscert(
+                                     &master_identity_key->pubkey,
+                                     rsa_identity_key,
+                                     time(NULL)+86400,
+                                     &rsa_ed_crosscert);
+
+  return;
+
+ err:
+  routerkeys_free_all();
+  tor_assert_nonfatal_unreached();
+}
+#undef MAKEKEY
+#undef MAKECERT
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Print the ISO8601-formated <b>expiration</b> for a certificate with
+ * some <b>description</b> to stdout.
+ *
+ * For example, for a signing certificate, this might print out:
+ * signing-cert-expiry: 2017-07-25 08:30:15 UTC
+ */
+static void
+print_cert_expiration(const char *expiration,
+                      const char *description)
+{
+  fprintf(stderr, "%s-cert-expiry: %s\n", description, expiration);
+}
+
+/**
+ * Log when a certificate, <b>cert</b>, with some <b>description</b> and
+ * stored in a file named <b>fname</b>, is going to expire.
+ */
+static void
+log_ed_cert_expiration(const tor_cert_t *cert,
+                       const char *description,
+                       const char *fname) {
+  char expiration[ISO_TIME_LEN+1];
+
+  if (BUG(!cert)) { /* If the specified key hasn't been loaded */
+    log_warn(LD_OR, "No %s key loaded; can't get certificate expiration.",
+             description);
+  } else {
+    format_local_iso_time(expiration, cert->valid_until);
+    log_notice(LD_OR, "The %s certificate stored in %s is valid until %s.",
+               description, fname, expiration);
+    print_cert_expiration(expiration, description);
+  }
+}
+
+/**
+ * Log when our master signing key certificate expires.  Used when tor is given
+ * the --key-expiration command-line option.
+ *
+ * Returns 0 on success and 1 on failure.
+ */
+static int
+log_master_signing_key_cert_expiration(const or_options_t *options)
+{
+  const tor_cert_t *signing_key;
+  char *fn = NULL;
+  int failed = 0;
+  time_t now = approx_time();
+
+  fn = options_get_keydir_fname(options, "ed25519_signing_cert");
+
+  /* Try to grab our cached copy of the key. */
+  signing_key = get_master_signing_key_cert();
+
+  tor_assert(server_identity_key_is_set());
+
+  /* Load our keys from disk, if necessary. */
+  if (!signing_key) {
+    failed = load_ed_keys(options, now) < 0;
+    signing_key = get_master_signing_key_cert();
+  }
+
+  /* If we do have a signing key, log the expiration time. */
+  if (signing_key) {
+    log_ed_cert_expiration(signing_key, "signing", fn);
+  } else {
+    log_warn(LD_OR, "Could not load signing key certificate from %s, so " \
+             "we couldn't learn anything about certificate expiration.", fn);
+  }
+
+  tor_free(fn);
+
+  return failed;
+}
+
+/**
+ * Log when a key certificate expires.  Used when tor is given the
+ * --key-expiration command-line option.
+ *
+ * If an command argument is given, which should specify the type of
+ * key to get expiry information about (currently supported arguments
+ * are "sign"), get info about that type of certificate.  Otherwise,
+ * print info about the supported arguments.
+ *
+ * Returns 0 on success and -1 on failure.
+ */
+int
+log_cert_expiration(void)
+{
+  const or_options_t *options = get_options();
+  const char *arg = options->command_arg;
+
+  if (!strcmp(arg, "sign")) {
+    return log_master_signing_key_cert_expiration(options);
+  } else {
+    fprintf(stderr, "No valid argument to --key-expiration found!\n");
+    fprintf(stderr, "Currently recognised arguments are: 'sign'\n");
+
+    return -1;
+  }
+}
+
+const ed25519_public_key_t *
+get_master_identity_key(void)
+{
+  if (!master_identity_key)
+    return NULL;
+  return &master_identity_key->pubkey;
+}
+
+/** Return true iff <b>id</b> is our Ed25519 master identity key. */
+int
+router_ed25519_id_is_me(const ed25519_public_key_t *id)
+{
+  return id && master_identity_key &&
+    ed25519_pubkey_eq(id, &master_identity_key->pubkey);
+}
+
+#ifdef TOR_UNIT_TESTS
+/* only exists for the unit tests, since otherwise the identity key
+ * should be used to sign nothing but the signing key. */
+const ed25519_keypair_t *
+get_master_identity_keypair(void)
+{
+  return master_identity_key;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+const ed25519_keypair_t *
+get_master_signing_keypair(void)
+{
+  return master_signing_key;
+}
+
+const struct tor_cert_st *
+get_master_signing_key_cert(void)
+{
+  return signing_key_cert;
+}
+
+const ed25519_keypair_t *
+get_current_auth_keypair(void)
+{
+  return current_auth_key;
+}
+
+const tor_cert_t *
+get_current_link_cert_cert(void)
+{
+  return link_cert_cert;
+}
+
+const tor_cert_t *
+get_current_auth_key_cert(void)
+{
+  return auth_key_cert;
+}
+
+void
+get_master_rsa_crosscert(const uint8_t **cert_out,
+                         size_t *size_out)
+{
+  *cert_out = rsa_ed_crosscert;
+  *size_out = rsa_ed_crosscert_len;
+}
+
+/** Construct cross-certification for the master identity key with
+ * the ntor onion key. Store the sign of the corresponding ed25519 public key
+ * in *<b>sign_out</b>. */
+tor_cert_t *
+make_ntor_onion_key_crosscert(const curve25519_keypair_t *onion_key,
+      const ed25519_public_key_t *master_id_key, time_t now, time_t lifetime,
+      int *sign_out)
+{
+  tor_cert_t *cert = NULL;
+  ed25519_keypair_t ed_onion_key;
+
+  if (ed25519_keypair_from_curve25519_keypair(&ed_onion_key, sign_out,
+                                              onion_key) < 0)
+    goto end;
+
+  cert = tor_cert_create(&ed_onion_key, CERT_TYPE_ONION_ID, master_id_key,
+      now, lifetime, 0);
+
+ end:
+  memwipe(&ed_onion_key, 0, sizeof(ed_onion_key));
+  return cert;
+}
+
+/** Construct and return an RSA signature for the TAP onion key to
+ * cross-certify the RSA and Ed25519 identity keys. Set <b>len_out</b> to its
+ * length. */
+uint8_t *
+make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
+                             const ed25519_public_key_t *master_id_key,
+                             const crypto_pk_t *rsa_id_key,
+                             int *len_out)
+{
+  uint8_t signature[PK_BYTES];
+  uint8_t signed_data[DIGEST_LEN + ED25519_PUBKEY_LEN];
+
+  *len_out = 0;
+  if (crypto_pk_get_digest(rsa_id_key, (char*)signed_data) < 0) {
+    return NULL;
+  }
+  memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN);
+
+  int r = crypto_pk_private_sign(onion_key,
+                               (char*)signature, sizeof(signature),
+                               (const char*)signed_data, sizeof(signed_data));
+  if (r < 0)
+    return NULL;
+
+  *len_out = r;
+
+  return tor_memdup(signature, r);
+}
+
+/** Check whether an RSA-TAP cross-certification is correct. Return 0 if it
+ * is, -1 if it isn't. */
+MOCK_IMPL(int,
+check_tap_onion_key_crosscert,(const uint8_t *crosscert,
+                               int crosscert_len,
+                               const crypto_pk_t *onion_pkey,
+                               const ed25519_public_key_t *master_id_pkey,
+                               const uint8_t *rsa_id_digest))
+{
+  uint8_t *cc = tor_malloc(crypto_pk_keysize(onion_pkey));
+  int cc_len =
+    crypto_pk_public_checksig(onion_pkey,
+                              (char*)cc,
+                              crypto_pk_keysize(onion_pkey),
+                              (const char*)crosscert,
+                              crosscert_len);
+  if (cc_len < 0) {
+    goto err;
+  }
+  if (cc_len < DIGEST_LEN + ED25519_PUBKEY_LEN) {
+    log_warn(LD_DIR, "Short signature on cross-certification with TAP key");
+    goto err;
+  }
+  if (tor_memneq(cc, rsa_id_digest, DIGEST_LEN) ||
+      tor_memneq(cc + DIGEST_LEN, master_id_pkey->pubkey,
+                 ED25519_PUBKEY_LEN)) {
+    log_warn(LD_DIR, "Incorrect cross-certification with TAP key");
+    goto err;
+  }
+
+  tor_free(cc);
+  return 0;
+ err:
+  tor_free(cc);
+  return -1;
+}
+
+void
+routerkeys_free_all(void)
+{
+  ed25519_keypair_free(master_identity_key);
+  ed25519_keypair_free(master_signing_key);
+  ed25519_keypair_free(current_auth_key);
+  tor_cert_free(signing_key_cert);
+  tor_cert_free(link_cert_cert);
+  tor_cert_free(auth_key_cert);
+  tor_free(rsa_ed_crosscert);
+
+  master_identity_key = master_signing_key = NULL;
+  current_auth_key = NULL;
+  signing_key_cert = link_cert_cert = auth_key_cert = NULL;
+  rsa_ed_crosscert = NULL; // redundant
+  rsa_ed_crosscert_len = 0;
+}
diff --git a/src/feature/relay/routerkeys.h b/src/feature/relay/routerkeys.h
new file mode 100644
index 0000000000..a6f06f6e20
--- /dev/null
+++ b/src/feature/relay/routerkeys.h
@@ -0,0 +1,85 @@
+/* Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_ROUTERKEYS_H
+#define TOR_ROUTERKEYS_H
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#define INIT_ED_KEY_CREATE                      (1u<<0)
+#define INIT_ED_KEY_REPLACE                     (1u<<1)
+#define INIT_ED_KEY_SPLIT                       (1u<<2)
+#define INIT_ED_KEY_MISSING_SECRET_OK           (1u<<3)
+#define INIT_ED_KEY_NEEDCERT                    (1u<<4)
+#define INIT_ED_KEY_EXTRA_STRONG                (1u<<5)
+#define INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT (1u<<6)
+#define INIT_ED_KEY_OMIT_SECRET                 (1u<<7)
+#define INIT_ED_KEY_TRY_ENCRYPTED               (1u<<8)
+#define INIT_ED_KEY_NO_REPAIR                   (1u<<9)
+#define INIT_ED_KEY_SUGGEST_KEYGEN              (1u<<10)
+#define INIT_ED_KEY_OFFLINE_SECRET              (1u<<11)
+#define INIT_ED_KEY_EXPLICIT_FNAME              (1u<<12)
+
+struct tor_cert_st;
+ed25519_keypair_t *ed_key_init_from_file(const char *fname, uint32_t flags,
+                                         int severity,
+                                         const ed25519_keypair_t *signing_key,
+                                         time_t now,
+                                         time_t lifetime,
+                                         uint8_t cert_type,
+                                         struct tor_cert_st **cert_out);
+ed25519_keypair_t *ed_key_new(const ed25519_keypair_t *signing_key,
+                              uint32_t flags,
+                              time_t now,
+                              time_t lifetime,
+                              uint8_t cert_type,
+                              struct tor_cert_st **cert_out);
+const ed25519_public_key_t *get_master_identity_key(void);
+const ed25519_keypair_t *get_master_signing_keypair(void);
+const struct tor_cert_st *get_master_signing_key_cert(void);
+
+const ed25519_keypair_t *get_current_auth_keypair(void);
+const struct tor_cert_st *get_current_link_cert_cert(void);
+const struct tor_cert_st *get_current_auth_key_cert(void);
+
+void get_master_rsa_crosscert(const uint8_t **cert_out,
+                              size_t *size_out);
+
+int router_ed25519_id_is_me(const ed25519_public_key_t *id);
+
+struct tor_cert_st *make_ntor_onion_key_crosscert(
+                                  const curve25519_keypair_t *onion_key,
+                                  const ed25519_public_key_t *master_id_key,
+                                  time_t now, time_t lifetime,
+                                  int *sign_out);
+uint8_t *make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
+                                  const ed25519_public_key_t *master_id_key,
+                                  const crypto_pk_t *rsa_id_key,
+                                  int *len_out);
+
+MOCK_DECL(int, check_tap_onion_key_crosscert,(const uint8_t *crosscert,
+                                  int crosscert_len,
+                                  const crypto_pk_t *onion_pkey,
+                                  const ed25519_public_key_t *master_id_pkey,
+                                  const uint8_t *rsa_id_digest));
+
+int log_cert_expiration(void);
+int load_ed_keys(const or_options_t *options, time_t now);
+int should_make_new_ed_keys(const or_options_t *options, const time_t now);
+
+int generate_ed_link_cert(const or_options_t *options, time_t now, int force);
+
+int read_encrypted_secret_key(ed25519_secret_key_t *out,
+                              const char *fname);
+int write_encrypted_secret_key(const ed25519_secret_key_t *out,
+                               const char *fname);
+
+void routerkeys_free_all(void);
+
+#ifdef TOR_UNIT_TESTS
+const ed25519_keypair_t *get_master_identity_keypair(void);
+void init_mock_ed_keys(const crypto_pk_t *rsa_identity_key);
+#endif
+
+#endif /* !defined(TOR_ROUTERKEYS_H) */
+
diff --git a/src/feature/rend/rend_authorized_client_st.h b/src/feature/rend/rend_authorized_client_st.h
new file mode 100644
index 0000000000..7ccf9771e1
--- /dev/null
+++ b/src/feature/rend/rend_authorized_client_st.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef REND_AUTHORIZED_CLIENT_ST_H
+#define REND_AUTHORIZED_CLIENT_ST_H
+
+/** Hidden-service side configuration of client authorization. */
+struct rend_authorized_client_t {
+  char *client_name;
+  uint8_t descriptor_cookie[REND_DESC_COOKIE_LEN];
+  crypto_pk_t *client_key;
+};
+
+#endif
+
diff --git a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
new file mode 100644
index 0000000000..0555ef6728
--- /dev/null
+++ b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H
+#define REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H
+
+/** ASCII-encoded v2 hidden service descriptor. */
+struct rend_encoded_v2_service_descriptor_t {
+  char desc_id[DIGEST_LEN]; /**< Descriptor ID. */
+  char *desc_str; /**< Descriptor string. */
+};
+
+#endif
+
diff --git a/src/feature/rend/rend_intro_point_st.h b/src/feature/rend/rend_intro_point_st.h
new file mode 100644
index 0000000000..89fe5ef2b3
--- /dev/null
+++ b/src/feature/rend/rend_intro_point_st.h
@@ -0,0 +1,76 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef REND_INTRO_POINT_ST_H
+#define REND_INTRO_POINT_ST_H
+
+struct replaycache_t;
+struct crypto_pk_t;
+
+/** Introduction point information.  Used both in rend_service_t (on
+ * the service side) and in rend_service_descriptor_t (on both the
+ * client and service side). */
+struct rend_intro_point_t {
+  extend_info_t *extend_info; /**< Extend info for connecting to this
+                               * introduction point via a multi-hop path. */
+  struct crypto_pk_t *intro_key; /**< Introduction key that replaces the
+                                  * service key, if this descriptor is V2. */
+
+  /** (Client side only) Flag indicating that a timeout has occurred
+   * after sending an INTRODUCE cell to this intro point.  After a
+   * timeout, an intro point should not be tried again during the same
+   * hidden service connection attempt, but it may be tried again
+   * during a future connection attempt. */
+  unsigned int timed_out : 1;
+
+  /** (Client side only) The number of times we have failed to build a
+   * circuit to this intro point for some reason other than our
+   * circuit-build timeout.  See also MAX_INTRO_POINT_REACHABILITY_FAILURES. */
+  unsigned int unreachable_count : 3;
+
+  /** (Service side only) Flag indicating that this intro point was
+   * included in the last HS descriptor we generated. */
+  unsigned int listed_in_last_desc : 1;
+
+  /** (Service side only) A replay cache recording the RSA-encrypted parts
+   * of INTRODUCE2 cells this intro point's circuit has received.  This is
+   * used to prevent replay attacks. */
+  struct replaycache_t *accepted_intro_rsa_parts;
+
+  /** (Service side only) Count of INTRODUCE2 cells accepted from this
+   * intro point.
+   */
+  int accepted_introduce2_count;
+
+  /** (Service side only) Maximum number of INTRODUCE2 cells that this IP
+   * will accept. This is a random value between
+   * INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS and
+   * INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS. */
+  int max_introductions;
+
+  /** (Service side only) The time at which this intro point was first
+   * published, or -1 if this intro point has not yet been
+   * published. */
+  time_t time_published;
+
+  /** (Service side only) The time at which this intro point should
+   * (start to) expire, or -1 if we haven't decided when this intro
+   * point should expire. */
+  time_t time_to_expire;
+
+  /** (Service side only) The amount of circuit creation we've made to this
+   * intro point. This is incremented every time we do a circuit relaunch on
+   * this object which is triggered when the circuit dies but the node is
+   * still in the consensus. After MAX_INTRO_POINT_CIRCUIT_RETRIES, we give
+   * up on it. */
+  unsigned int circuit_retries;
+
+  /** (Service side only) Set if this intro point has an established circuit
+   * and unset if it doesn't. */
+  unsigned int circuit_established:1;
+};
+
+#endif
diff --git a/src/feature/rend/rend_service_descriptor_st.h b/src/feature/rend/rend_service_descriptor_st.h
new file mode 100644
index 0000000000..8ea8a62305
--- /dev/null
+++ b/src/feature/rend/rend_service_descriptor_st.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef REND_SERVICE_DESCRIPTOR_ST_H
+#define REND_SERVICE_DESCRIPTOR_ST_H
+
+#define REND_PROTOCOL_VERSION_BITMASK_WIDTH 16
+
+/** Information used to connect to a hidden service.  Used on both the
+ * service side and the client side. */
+struct rend_service_descriptor_t {
+  crypto_pk_t *pk; /**< This service's public key. */
+  int version; /**< Version of the descriptor format: 0 or 2. */
+  time_t timestamp; /**< Time when the descriptor was generated. */
+  /** Bitmask: which introduce/rendezvous protocols are supported?
+   * (We allow bits '0', '1', '2' and '3' to be set.) */
+  unsigned protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH;
+  /** List of the service's introduction points.  Elements are removed if
+   * introduction attempts fail. */
+  smartlist_t *intro_nodes;
+  /** Has descriptor been uploaded to all hidden service directories? */
+  int all_uploads_performed;
+  /** List of hidden service directories to which an upload request for
+   * this descriptor could be sent. Smartlist exists only when at least one
+   * of the previous upload requests failed (otherwise it's not important
+   * to know which uploads succeeded and which not). */
+  smartlist_t *successful_uploads;
+};
+
+#endif
+
diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c
new file mode 100644
index 0000000000..c18920154e
--- /dev/null
+++ b/src/feature/rend/rendcache.c
@@ -0,0 +1,1008 @@
+/* Copyright (c) 2015-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendcache.c
+ * \brief Hidden service descriptor cache.
+ **/
+
+#define RENDCACHE_PRIVATE
+#include "or/rendcache.h"
+
+#include "or/config.h"
+#include "or/rephist.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/rendcommon.h"
+
+#include "or/extend_info_st.h"
+#include "or/rend_intro_point_st.h"
+#include "or/rend_service_descriptor_st.h"
+
+/** Map from service id (as generated by rend_get_service_id) to
+ * rend_cache_entry_t. */
+STATIC strmap_t *rend_cache = NULL;
+
+/** Map from service id to rend_cache_entry_t; only for hidden services. */
+static strmap_t *rend_cache_local_service = NULL;
+
+/** Map from descriptor id to rend_cache_entry_t; only for hidden service
+ * directories. */
+STATIC digestmap_t *rend_cache_v2_dir = NULL;
+
+/** (Client side only) Map from service id to rend_cache_failure_t. This
+ * cache is used to track intro point(IP) failures so we know when to keep
+ * or discard a new descriptor we just fetched. Here is a description of the
+ * cache behavior.
+ *
+ * Everytime tor discards an IP (ex: receives a NACK), we add an entry to
+ * this cache noting the identity digest of the IP and it's failure type for
+ * the service ID. The reason we indexed this cache by service ID is to
+ * differentiate errors that can occur only for a specific service like a
+ * NACK for instance. It applies for one but maybe not for the others.
+ *
+ * Once a service descriptor is fetched and considered valid, each IP is
+ * looked up in this cache and if present, it is discarded from the fetched
+ * descriptor. At the end, all IP(s) in the cache, for a specific service
+ * ID, that were NOT present in the descriptor are removed from this cache.
+ * Which means that if at least one IP was not in this cache, thus usuable,
+ * it's considered a new descriptor so we keep it. Else, if all IPs were in
+ * this cache, we discard the descriptor as it's considered unusable.
+ *
+ * Once a descriptor is removed from the rend cache or expires, the entry
+ * in this cache is also removed for the service ID.
+ *
+ * This scheme allows us to not rely on the descriptor's timestamp (which
+ * is rounded down to the hour) to know if we have a newer descriptor. We
+ * only rely on the usability of intro points from an internal state. */
+STATIC strmap_t *rend_cache_failure = NULL;
+
+/* DOCDOC */
+STATIC size_t rend_cache_total_allocation = 0;
+
+/** Initializes the service descriptor cache.
+*/
+void
+rend_cache_init(void)
+{
+  rend_cache = strmap_new();
+  rend_cache_v2_dir = digestmap_new();
+  rend_cache_local_service = strmap_new();
+  rend_cache_failure = strmap_new();
+}
+
+/** Return the approximate number of bytes needed to hold <b>e</b>. */
+STATIC size_t
+rend_cache_entry_allocation(const rend_cache_entry_t *e)
+{
+  if (!e)
+    return 0;
+
+  /* This doesn't count intro_nodes or key size */
+  return sizeof(*e) + e->len + sizeof(*e->parsed);
+}
+
+/* DOCDOC */
+size_t
+rend_cache_get_total_allocation(void)
+{
+  return rend_cache_total_allocation;
+}
+
+/** Decrement the total bytes attributed to the rendezvous cache by n. */
+void
+rend_cache_decrement_allocation(size_t n)
+{
+  static int have_underflowed = 0;
+
+  if (rend_cache_total_allocation >= n) {
+    rend_cache_total_allocation -= n;
+  } else {
+    rend_cache_total_allocation = 0;
+    if (! have_underflowed) {
+      have_underflowed = 1;
+      log_warn(LD_BUG, "Underflow in rend_cache_decrement_allocation");
+    }
+  }
+}
+
+/** Increase the total bytes attributed to the rendezvous cache by n. */
+void
+rend_cache_increment_allocation(size_t n)
+{
+  static int have_overflowed = 0;
+  if (rend_cache_total_allocation <= SIZE_MAX - n) {
+    rend_cache_total_allocation += n;
+  } else {
+    rend_cache_total_allocation = SIZE_MAX;
+    if (! have_overflowed) {
+      have_overflowed = 1;
+      log_warn(LD_BUG, "Overflow in rend_cache_increment_allocation");
+    }
+  }
+}
+
+/** Helper: free a rend cache failure intro object. */
+STATIC void
+rend_cache_failure_intro_entry_free_(rend_cache_failure_intro_t *entry)
+{
+  if (entry == NULL) {
+    return;
+  }
+  tor_free(entry);
+}
+
+static void
+rend_cache_failure_intro_entry_free_void(void *entry)
+{
+  rend_cache_failure_intro_entry_free_(entry);
+}
+
+/** Allocate a rend cache failure intro object and return it. <b>failure</b>
+ * is set into the object. This function can not fail. */
+STATIC rend_cache_failure_intro_t *
+rend_cache_failure_intro_entry_new(rend_intro_point_failure_t failure)
+{
+  rend_cache_failure_intro_t *entry = tor_malloc(sizeof(*entry));
+  entry->failure_type = failure;
+  entry->created_ts = time(NULL);
+  return entry;
+}
+
+/** Helper: free a rend cache failure object. */
+STATIC void
+rend_cache_failure_entry_free_(rend_cache_failure_t *entry)
+{
+  if (entry == NULL) {
+    return;
+  }
+
+  /* Free and remove every intro failure object. */
+  digestmap_free(entry->intro_failures,
+                 rend_cache_failure_intro_entry_free_void);
+
+  tor_free(entry);
+}
+
+/** Helper: deallocate a rend_cache_failure_t. (Used with strmap_free(),
+ * which requires a function pointer whose argument is void*). */
+STATIC void
+rend_cache_failure_entry_free_void(void *entry)
+{
+  rend_cache_failure_entry_free_(entry);
+}
+
+/** Allocate a rend cache failure object and return it. This function can
+ * not fail. */
+STATIC rend_cache_failure_t *
+rend_cache_failure_entry_new(void)
+{
+  rend_cache_failure_t *entry = tor_malloc(sizeof(*entry));
+  entry->intro_failures = digestmap_new();
+  return entry;
+}
+
+/** Remove failure cache entry for the service ID in the given descriptor
+ * <b>desc</b>. */
+STATIC void
+rend_cache_failure_remove(rend_service_descriptor_t *desc)
+{
+  char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
+  rend_cache_failure_t *entry;
+
+  if (desc == NULL) {
+    return;
+  }
+  if (rend_get_service_id(desc->pk, service_id) < 0) {
+    return;
+  }
+  entry = strmap_get_lc(rend_cache_failure, service_id);
+  if (entry != NULL) {
+    strmap_remove_lc(rend_cache_failure, service_id);
+    rend_cache_failure_entry_free(entry);
+  }
+}
+
+/** Helper: free storage held by a single service descriptor cache entry. */
+STATIC void
+rend_cache_entry_free_(rend_cache_entry_t *e)
+{
+  if (!e)
+    return;
+  rend_cache_decrement_allocation(rend_cache_entry_allocation(e));
+  /* We are about to remove a descriptor from the cache so remove the entry
+   * in the failure cache. */
+  rend_cache_failure_remove(e->parsed);
+  rend_service_descriptor_free(e->parsed);
+  tor_free(e->desc);
+  tor_free(e);
+}
+
+/** Helper: deallocate a rend_cache_entry_t.  (Used with strmap_free(), which
+ * requires a function pointer whose argument is void*). */
+static void
+rend_cache_entry_free_void(void *p)
+{
+  rend_cache_entry_free_(p);
+}
+
+/** Free all storage held by the service descriptor cache. */
+void
+rend_cache_free_all(void)
+{
+  strmap_free(rend_cache, rend_cache_entry_free_void);
+  digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_void);
+  strmap_free(rend_cache_local_service, rend_cache_entry_free_void);
+  strmap_free(rend_cache_failure, rend_cache_failure_entry_free_void);
+  rend_cache = NULL;
+  rend_cache_v2_dir = NULL;
+  rend_cache_local_service = NULL;
+  rend_cache_failure = NULL;
+  rend_cache_total_allocation = 0;
+}
+
+/** Remove all entries that re REND_CACHE_FAILURE_MAX_AGE old. This is
+ * called every second.
+ *
+ * We have to clean these regurlarly else if for whatever reasons an hidden
+ * service goes offline and a client tries to connect to it during that
+ * time, a failure entry is created and the client will be unable to connect
+ * for a while even though the service has return online.  */
+void
+rend_cache_failure_clean(time_t now)
+{
+  time_t cutoff = now - REND_CACHE_FAILURE_MAX_AGE;
+  STRMAP_FOREACH_MODIFY(rend_cache_failure, key,
+                        rend_cache_failure_t *, ent) {
+    /* Free and remove every intro failure object that match the cutoff. */
+    DIGESTMAP_FOREACH_MODIFY(ent->intro_failures, ip_key,
+                             rend_cache_failure_intro_t *, ip_ent) {
+      if (ip_ent->created_ts < cutoff) {
+        rend_cache_failure_intro_entry_free(ip_ent);
+        MAP_DEL_CURRENT(ip_key);
+      }
+    } DIGESTMAP_FOREACH_END;
+    /* If the entry is now empty of intro point failures, remove it. */
+    if (digestmap_isempty(ent->intro_failures)) {
+      rend_cache_failure_entry_free(ent);
+      MAP_DEL_CURRENT(key);
+    }
+  } STRMAP_FOREACH_END;
+}
+
+/** Removes all old entries from the client or service descriptor cache.
+*/
+void
+rend_cache_clean(time_t now, rend_cache_type_t cache_type)
+{
+  strmap_iter_t *iter;
+  const char *key;
+  void *val;
+  rend_cache_entry_t *ent;
+  time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW;
+  strmap_t *cache = NULL;
+
+  if (cache_type == REND_CACHE_TYPE_CLIENT) {
+    cache = rend_cache;
+  } else if (cache_type == REND_CACHE_TYPE_SERVICE)  {
+    cache = rend_cache_local_service;
+  }
+  tor_assert(cache);
+
+  for (iter = strmap_iter_init(cache); !strmap_iter_done(iter); ) {
+    strmap_iter_get(iter, &key, &val);
+    ent = (rend_cache_entry_t*)val;
+    if (ent->parsed->timestamp < cutoff) {
+      iter = strmap_iter_next_rmv(cache, iter);
+      rend_cache_entry_free(ent);
+    } else {
+      iter = strmap_iter_next(cache, iter);
+    }
+  }
+}
+
+/** Remove ALL entries from the rendezvous service descriptor cache.
+*/
+void
+rend_cache_purge(void)
+{
+  if (rend_cache) {
+    log_info(LD_REND, "Purging HS v2 descriptor cache");
+    strmap_free(rend_cache, rend_cache_entry_free_void);
+  }
+  rend_cache = strmap_new();
+}
+
+/** Remove ALL entries from the failure cache. This is also called when a
+ * NEWNYM signal is received. */
+void
+rend_cache_failure_purge(void)
+{
+  if (rend_cache_failure) {
+    log_info(LD_REND, "Purging HS v2 failure cache");
+    strmap_free(rend_cache_failure, rend_cache_failure_entry_free_void);
+  }
+  rend_cache_failure = strmap_new();
+}
+
+/** Lookup the rend failure cache using a relay identity digest in
+ * <b>identity</b> which has DIGEST_LEN bytes and service ID <b>service_id</b>
+ * which is a null-terminated string. If found, the intro failure is set in
+ * <b>intro_entry</b> else it stays untouched. Return 1 iff found else 0. */
+STATIC int
+cache_failure_intro_lookup(const uint8_t *identity, const char *service_id,
+                           rend_cache_failure_intro_t **intro_entry)
+{
+  rend_cache_failure_t *elem;
+  rend_cache_failure_intro_t *intro_elem;
+
+  tor_assert(rend_cache_failure);
+
+  if (intro_entry) {
+    *intro_entry = NULL;
+  }
+
+  /* Lookup descriptor and return it. */
+  elem = strmap_get_lc(rend_cache_failure, service_id);
+  if (elem == NULL) {
+    goto not_found;
+  }
+  intro_elem = digestmap_get(elem->intro_failures, (char *) identity);
+  if (intro_elem == NULL) {
+    goto not_found;
+  }
+  if (intro_entry) {
+    *intro_entry = intro_elem;
+  }
+  return 1;
+ not_found:
+  return 0;
+}
+
+/** Allocate a new cache failure intro object and copy the content from
+ * <b>entry</b> to this newly allocated object. Return it. */
+static rend_cache_failure_intro_t *
+cache_failure_intro_dup(const rend_cache_failure_intro_t *entry)
+{
+  rend_cache_failure_intro_t *ent_dup =
+    rend_cache_failure_intro_entry_new(entry->failure_type);
+  ent_dup->created_ts = entry->created_ts;
+  return ent_dup;
+}
+
+/** Add an intro point failure to the failure cache using the relay
+ * <b>identity</b> and service ID <b>service_id</b>. Record the
+ * <b>failure</b> in that object. */
+STATIC void
+cache_failure_intro_add(const uint8_t *identity, const char *service_id,
+                        rend_intro_point_failure_t failure)
+{
+  rend_cache_failure_t *fail_entry;
+  rend_cache_failure_intro_t *entry, *old_entry;
+
+  /* Make sure we have a failure object for this service ID and if not,
+   * create it with this new intro failure entry. */
+  fail_entry = strmap_get_lc(rend_cache_failure, service_id);
+  if (fail_entry == NULL) {
+    fail_entry = rend_cache_failure_entry_new();
+    /* Add failure entry to global rend failure cache. */
+    strmap_set_lc(rend_cache_failure, service_id, fail_entry);
+  }
+  entry = rend_cache_failure_intro_entry_new(failure);
+  old_entry = digestmap_set(fail_entry->intro_failures,
+                            (char *) identity, entry);
+  /* This _should_ be NULL, but in case it isn't, free it. */
+  rend_cache_failure_intro_entry_free(old_entry);
+}
+
+/** Using a parsed descriptor <b>desc</b>, check if the introduction points
+ * are present in the failure cache and if so they are removed from the
+ * descriptor and kept into the failure cache. Then, each intro points that
+ * are NOT in the descriptor but in the failure cache for the given
+ * <b>service_id</b> are removed from the failure cache. */
+STATIC void
+validate_intro_point_failure(const rend_service_descriptor_t *desc,
+                             const char *service_id)
+{
+  rend_cache_failure_t *new_entry, *cur_entry;
+  /* New entry for the service ID that will be replacing the one in the
+   * failure cache since we have a new descriptor. In the case where all
+   * intro points are removed, we are assured that the new entry is the same
+   * as the current one. */
+  new_entry = tor_malloc(sizeof(*new_entry));
+  new_entry->intro_failures = digestmap_new();
+
+  tor_assert(desc);
+
+  SMARTLIST_FOREACH_BEGIN(desc->intro_nodes, rend_intro_point_t *, intro) {
+    int found;
+    rend_cache_failure_intro_t *entry;
+    const uint8_t *identity =
+      (uint8_t *) intro->extend_info->identity_digest;
+
+    found = cache_failure_intro_lookup(identity, service_id, &entry);
+    if (found) {
+      /* Dup here since it will be freed at the end when removing the
+       * original entry in the cache. */
+      rend_cache_failure_intro_t *ent_dup = cache_failure_intro_dup(entry);
+      /* This intro point is in our cache, discard it from the descriptor
+       * because chances are that it's unusable. */
+      SMARTLIST_DEL_CURRENT(desc->intro_nodes, intro);
+      /* Keep it for our new entry. */
+      digestmap_set(new_entry->intro_failures, (char *) identity, ent_dup);
+      /* Only free it when we're done looking at it. */
+      rend_intro_point_free(intro);
+      continue;
+    }
+  } SMARTLIST_FOREACH_END(intro);
+
+  /* Swap the failure entry in the cache and free the current one. */
+  cur_entry = strmap_get_lc(rend_cache_failure, service_id);
+  if (cur_entry != NULL) {
+    rend_cache_failure_entry_free(cur_entry);
+  }
+  strmap_set_lc(rend_cache_failure, service_id, new_entry);
+}
+
+/** Note down an intro failure in the rend failure cache using the type of
+ * failure in <b>failure</b> for the relay identity digest in
+ * <b>identity</b> and service ID <b>service_id</b>. If an entry already
+ * exists in the cache, the failure type is changed with <b>failure</b>. */
+void
+rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
+                              const uint8_t *identity,
+                              const char *service_id)
+{
+  int found;
+  rend_cache_failure_intro_t *entry;
+
+  found = cache_failure_intro_lookup(identity, service_id, &entry);
+  if (!found) {
+    cache_failure_intro_add(identity, service_id, failure);
+  } else {
+    /* Replace introduction point failure with this one. */
+    entry->failure_type = failure;
+  }
+}
+
+/** Remove all old v2 descriptors and those for which this hidden service
+ * directory is not responsible for any more. The cutoff is the time limit for
+ * which we want to keep the cache entry. In other words, any entry created
+ * before will be removed. */
+size_t
+rend_cache_clean_v2_descs_as_dir(time_t cutoff)
+{
+  digestmap_iter_t *iter;
+  size_t bytes_removed = 0;
+
+  for (iter = digestmap_iter_init(rend_cache_v2_dir);
+       !digestmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    rend_cache_entry_t *ent;
+    digestmap_iter_get(iter, &key, &val);
+    ent = val;
+    if (ent->parsed->timestamp < cutoff) {
+      char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+      base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN);
+      log_info(LD_REND, "Removing descriptor with ID '%s' from cache",
+               safe_str_client(key_base32));
+      bytes_removed += rend_cache_entry_allocation(ent);
+      iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter);
+      rend_cache_entry_free(ent);
+    } else {
+      iter = digestmap_iter_next(rend_cache_v2_dir, iter);
+    }
+  }
+
+  return bytes_removed;
+}
+
+/** Lookup in the client cache the given service ID <b>query</b> for
+ * <b>version</b>.
+ *
+ * Return 0 if found and if <b>e</b> is non NULL, set it with the entry
+ * found. Else, a negative value is returned and <b>e</b> is untouched.
+ * -EINVAL means that <b>query</b> is not a valid service id.
+ * -ENOENT means that no entry in the cache was found. */
+int
+rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e)
+{
+  int ret = 0;
+  char key[REND_SERVICE_ID_LEN_BASE32 + 2]; /* <version><query>\0 */
+  rend_cache_entry_t *entry = NULL;
+  static const int default_version = 2;
+
+  tor_assert(rend_cache);
+  tor_assert(query);
+
+  if (!rend_valid_v2_service_id(query)) {
+    ret = -EINVAL;
+    goto end;
+  }
+
+  switch (version) {
+    case 0:
+      log_warn(LD_REND, "Cache lookup of a v0 renddesc is deprecated.");
+      break;
+    case 2:
+      /* Default is version 2. */
+    default:
+      tor_snprintf(key, sizeof(key), "%d%s", default_version, query);
+      entry = strmap_get_lc(rend_cache, key);
+      break;
+  }
+  if (!entry) {
+    ret = -ENOENT;
+    goto end;
+  }
+  tor_assert(entry->parsed && entry->parsed->intro_nodes);
+
+  if (e) {
+    *e = entry;
+  }
+
+ end:
+  return ret;
+}
+
+/*
+ * Lookup the v2 service descriptor with the service ID <b>query</b> in the
+ * local service descriptor cache. Return 0 if found and if <b>e</b> is
+ * non NULL, set it with the entry found. Else, a negative value is returned
+ * and <b>e</b> is untouched.
+ * -EINVAL means that <b>query</b> is not a valid service id.
+ * -ENOENT means that no entry in the cache was found. */
+int
+rend_cache_lookup_v2_desc_as_service(const char *query, rend_cache_entry_t **e)
+{
+  int ret = 0;
+  rend_cache_entry_t *entry = NULL;
+
+  tor_assert(rend_cache_local_service);
+  tor_assert(query);
+
+  if (!rend_valid_v2_service_id(query)) {
+    ret = -EINVAL;
+    goto end;
+  }
+
+  /* Lookup descriptor and return. */
+  entry = strmap_get_lc(rend_cache_local_service, query);
+  if (!entry) {
+    ret = -ENOENT;
+    goto end;
+  }
+
+  if (e) {
+    *e = entry;
+  }
+
+ end:
+  return ret;
+}
+
+/** Lookup the v2 service descriptor with base32-encoded <b>desc_id</b> and
+ * copy the pointer to it to *<b>desc</b>.  Return 1 on success, 0 on
+ * well-formed-but-not-found, and -1 on failure.
+ */
+int
+rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc)
+{
+  rend_cache_entry_t *e;
+  char desc_id_digest[DIGEST_LEN];
+  tor_assert(rend_cache_v2_dir);
+  if (base32_decode(desc_id_digest, DIGEST_LEN,
+                    desc_id, REND_DESC_ID_V2_LEN_BASE32) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Rejecting v2 rendezvous descriptor request -- descriptor ID "
+           "contains illegal characters: %s",
+           safe_str(desc_id));
+    return -1;
+  }
+  /* Lookup descriptor and return. */
+  e = digestmap_get(rend_cache_v2_dir, desc_id_digest);
+  if (e) {
+    *desc = e->desc;
+    e->last_served = approx_time();
+    return 1;
+  }
+  return 0;
+}
+
+/** Parse the v2 service descriptor(s) in <b>desc</b> and store it/them to the
+ * local rend cache. Don't attempt to decrypt the included list of introduction
+ * points (as we don't have a descriptor cookie for it).
+ *
+ * If we have a newer descriptor with the same ID, ignore this one.
+ * If we have an older descriptor with the same ID, replace it.
+ *
+ * Return 0 on success, or -1 if we couldn't parse any of them.
+ *
+ * We should only call this function for public (e.g. non bridge) relays.
+ */
+int
+rend_cache_store_v2_desc_as_dir(const char *desc)
+{
+  const or_options_t *options = get_options();
+  rend_service_descriptor_t *parsed;
+  char desc_id[DIGEST_LEN];
+  char *intro_content;
+  size_t intro_size;
+  size_t encoded_size;
+  char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+  int number_parsed = 0, number_stored = 0;
+  const char *current_desc = desc;
+  const char *next_desc;
+  rend_cache_entry_t *e;
+  time_t now = time(NULL);
+  tor_assert(rend_cache_v2_dir);
+  tor_assert(desc);
+  while (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content,
+                                          &intro_size, &encoded_size,
+                                          &next_desc, current_desc, 1) >= 0) {
+    number_parsed++;
+    /* We don't care about the introduction points. */
+    tor_free(intro_content);
+    /* For pretty log statements. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32),
+                  desc_id, DIGEST_LEN);
+    /* Is descriptor too old? */
+    if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) {
+      log_info(LD_REND, "Service descriptor with desc ID %s is too old.",
+               safe_str(desc_id_base32));
+      goto skip;
+    }
+    /* Is descriptor too far in the future? */
+    if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) {
+      log_info(LD_REND, "Service descriptor with desc ID %s is too far in the "
+               "future.",
+               safe_str(desc_id_base32));
+      goto skip;
+    }
+    /* Do we already have a newer descriptor? */
+    e = digestmap_get(rend_cache_v2_dir, desc_id);
+    if (e && e->parsed->timestamp > parsed->timestamp) {
+      log_info(LD_REND, "We already have a newer service descriptor with the "
+               "same desc ID %s and version.",
+               safe_str(desc_id_base32));
+      goto skip;
+    }
+    /* Do we already have this descriptor? */
+    if (e && !strcmp(desc, e->desc)) {
+      log_info(LD_REND, "We already have this service descriptor with desc "
+               "ID %s.", safe_str(desc_id_base32));
+      goto skip;
+    }
+    /* Store received descriptor. */
+    if (!e) {
+      e = tor_malloc_zero(sizeof(rend_cache_entry_t));
+      digestmap_set(rend_cache_v2_dir, desc_id, e);
+      /* Treat something just uploaded as having been served a little
+       * while ago, so that flooding with new descriptors doesn't help
+       * too much.
+       */
+      e->last_served = approx_time() - 3600;
+    } else {
+      rend_cache_decrement_allocation(rend_cache_entry_allocation(e));
+      rend_service_descriptor_free(e->parsed);
+      tor_free(e->desc);
+    }
+    e->parsed = parsed;
+    e->desc = tor_strndup(current_desc, encoded_size);
+    e->len = encoded_size;
+    rend_cache_increment_allocation(rend_cache_entry_allocation(e));
+    log_info(LD_REND, "Successfully stored service descriptor with desc ID "
+             "'%s' and len %d.",
+             safe_str(desc_id_base32), (int)encoded_size);
+    /* Statistics: Note down this potentially new HS. */
+    if (options->HiddenServiceStatistics) {
+      rep_hist_stored_maybe_new_hs(e->parsed->pk);
+    }
+
+    number_stored++;
+    goto advance;
+ skip:
+    rend_service_descriptor_free(parsed);
+ advance:
+    /* advance to next descriptor, if available. */
+    current_desc = next_desc;
+    /* check if there is a next descriptor. */
+    if (!current_desc ||
+        strcmpstart(current_desc, "rendezvous-service-descriptor "))
+      break;
+  }
+  if (!number_parsed) {
+    log_info(LD_REND, "Could not parse any descriptor.");
+    return -1;
+  }
+  log_info(LD_REND, "Parsed %d and added %d descriptor%s.",
+           number_parsed, number_stored, number_stored != 1 ? "s" : "");
+  return 0;
+}
+
+/** Parse the v2 service descriptor in <b>desc</b> and store it to the
+* local service rend cache. Don't attempt to decrypt the included list of
+* introduction points.
+*
+* If we have a newer descriptor with the same ID, ignore this one.
+* If we have an older descriptor with the same ID, replace it.
+*
+* Return 0 on success, or -1 if we couldn't understand the descriptor.
+*/
+int
+rend_cache_store_v2_desc_as_service(const char *desc)
+{
+  rend_service_descriptor_t *parsed = NULL;
+  char desc_id[DIGEST_LEN];
+  char *intro_content = NULL;
+  size_t intro_size;
+  size_t encoded_size;
+  const char *next_desc;
+  char service_id[REND_SERVICE_ID_LEN_BASE32+1];
+  rend_cache_entry_t *e;
+  int retval = -1;
+  tor_assert(rend_cache_local_service);
+  tor_assert(desc);
+
+  /* Parse the descriptor. */
+  if (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content,
+                                       &intro_size, &encoded_size,
+                                       &next_desc, desc, 0) < 0) {
+    log_warn(LD_REND, "Could not parse descriptor.");
+    goto err;
+  }
+  /* Compute service ID from public key. */
+  if (rend_get_service_id(parsed->pk, service_id)<0) {
+    log_warn(LD_REND, "Couldn't compute service ID.");
+    goto err;
+  }
+
+  /* Do we already have a newer descriptor? Allow new descriptors with a
+     rounded timestamp equal to or newer than the current descriptor */
+  e = (rend_cache_entry_t*) strmap_get_lc(rend_cache_local_service,
+                                          service_id);
+  if (e && e->parsed->timestamp > parsed->timestamp) {
+    log_info(LD_REND, "We already have a newer service descriptor for "
+             "service ID %s.", safe_str_client(service_id));
+    goto okay;
+  }
+  /* We don't care about the introduction points. */
+  tor_free(intro_content);
+  if (!e) {
+    e = tor_malloc_zero(sizeof(rend_cache_entry_t));
+    strmap_set_lc(rend_cache_local_service, service_id, e);
+  } else {
+    rend_cache_decrement_allocation(rend_cache_entry_allocation(e));
+    rend_service_descriptor_free(e->parsed);
+    tor_free(e->desc);
+  }
+  e->parsed = parsed;
+  e->desc = tor_malloc_zero(encoded_size + 1);
+  strlcpy(e->desc, desc, encoded_size + 1);
+  e->len = encoded_size;
+  rend_cache_increment_allocation(rend_cache_entry_allocation(e));
+  log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.",
+            safe_str_client(service_id), (int)encoded_size);
+  return 0;
+
+ okay:
+  retval = 0;
+
+ err:
+  rend_service_descriptor_free(parsed);
+  tor_free(intro_content);
+  return retval;
+}
+
+/** Parse the v2 service descriptor in <b>desc</b>, decrypt the included list
+ * of introduction points with <b>descriptor_cookie</b> (which may also be
+ * <b>NULL</b> if decryption is not necessary), and store the descriptor to
+ * the local cache under its version and service id.
+ *
+ * If we have a newer v2 descriptor with the same ID, ignore this one.
+ * If we have an older descriptor with the same ID, replace it.
+ * If the descriptor's service ID does not match
+ * <b>rend_query</b>-\>onion_address, reject it.
+ *
+ * If the descriptor's descriptor ID doesn't match <b>desc_id_base32</b>,
+ * reject it.
+ *
+ * Return 0 on success, or -1 if we rejected the descriptor.
+ * If entry is not NULL, set it with the cache entry pointer of the descriptor.
+ */
+int
+rend_cache_store_v2_desc_as_client(const char *desc,
+                                   const char *desc_id_base32,
+                                   const rend_data_t *rend_query,
+                                   rend_cache_entry_t **entry)
+{
+  /*XXXX this seems to have a bit of duplicate code with
+   * rend_cache_store_v2_desc_as_dir().  Fix that. */
+  /* Though having similar elements, both functions were separated on
+   * purpose:
+   * - dirs don't care about encoded/encrypted introduction points, clients
+   *   do.
+   * - dirs store descriptors in a separate cache by descriptor ID, whereas
+   *   clients store them by service ID; both caches are different data
+   *   structures and have different access methods.
+   * - dirs store a descriptor only if they are responsible for its ID,
+   *   clients do so in every way (because they have requested it before).
+   * - dirs can process multiple concatenated descriptors which is required
+   *   for replication, whereas clients only accept a single descriptor.
+   * Thus, combining both methods would result in a lot of if statements
+   * which probably would not improve, but worsen code readability. -KL */
+  rend_service_descriptor_t *parsed = NULL;
+  char desc_id[DIGEST_LEN];
+  char *intro_content = NULL;
+  size_t intro_size;
+  size_t encoded_size;
+  const char *next_desc;
+  time_t now = time(NULL);
+  char key[REND_SERVICE_ID_LEN_BASE32+2];
+  char service_id[REND_SERVICE_ID_LEN_BASE32+1];
+  char want_desc_id[DIGEST_LEN];
+  rend_cache_entry_t *e;
+  int retval = -1;
+  rend_data_v2_t *rend_data = TO_REND_DATA_V2(rend_query);
+
+  tor_assert(rend_cache);
+  tor_assert(desc);
+  tor_assert(desc_id_base32);
+  memset(want_desc_id, 0, sizeof(want_desc_id));
+  if (entry) {
+    *entry = NULL;
+  }
+  if (base32_decode(want_desc_id, sizeof(want_desc_id),
+                    desc_id_base32, strlen(desc_id_base32)) != 0) {
+    log_warn(LD_BUG, "Couldn't decode base32 %s for descriptor id.",
+             escaped_safe_str_client(desc_id_base32));
+    goto err;
+  }
+  /* Parse the descriptor. */
+  if (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content,
+                                       &intro_size, &encoded_size,
+                                       &next_desc, desc, 0) < 0) {
+    log_warn(LD_REND, "Could not parse descriptor.");
+    goto err;
+  }
+  /* Compute service ID from public key. */
+  if (rend_get_service_id(parsed->pk, service_id)<0) {
+    log_warn(LD_REND, "Couldn't compute service ID.");
+    goto err;
+  }
+  if (rend_data->onion_address[0] != '\0' &&
+      strcmp(rend_data->onion_address, service_id)) {
+    log_warn(LD_REND, "Received service descriptor for service ID %s; "
+             "expected descriptor for service ID %s.",
+             service_id, safe_str(rend_data->onion_address));
+    goto err;
+  }
+  if (tor_memneq(desc_id, want_desc_id, DIGEST_LEN)) {
+    log_warn(LD_REND, "Received service descriptor for %s with incorrect "
+             "descriptor ID.", service_id);
+    goto err;
+  }
+
+  /* Decode/decrypt introduction points. */
+  if (intro_content && intro_size > 0) {
+    int n_intro_points;
+    if (rend_data->auth_type != REND_NO_AUTH &&
+        !tor_mem_is_zero(rend_data->descriptor_cookie,
+                         sizeof(rend_data->descriptor_cookie))) {
+      char *ipos_decrypted = NULL;
+      size_t ipos_decrypted_size;
+      if (rend_decrypt_introduction_points(&ipos_decrypted,
+                                           &ipos_decrypted_size,
+                                           rend_data->descriptor_cookie,
+                                           intro_content,
+                                           intro_size) < 0) {
+        log_warn(LD_REND, "Failed to decrypt introduction points. We are "
+                 "probably unable to parse the encoded introduction points.");
+      } else {
+        /* Replace encrypted with decrypted introduction points. */
+        log_info(LD_REND, "Successfully decrypted introduction points.");
+        tor_free(intro_content);
+        intro_content = ipos_decrypted;
+        intro_size = ipos_decrypted_size;
+      }
+    }
+    n_intro_points = rend_parse_introduction_points(parsed, intro_content,
+                                                    intro_size);
+    if (n_intro_points <= 0) {
+      log_warn(LD_REND, "Failed to parse introduction points. Either the "
+               "service has published a corrupt descriptor or you have "
+               "provided invalid authorization data.");
+      goto err;
+    } else if (n_intro_points > MAX_INTRO_POINTS) {
+      log_warn(LD_REND, "Found too many introduction points on a hidden "
+               "service descriptor for %s. This is probably a (misguided) "
+               "attempt to improve reliability, but it could also be an "
+               "attempt to do a guard enumeration attack. Rejecting.",
+               safe_str_client(service_id));
+
+      goto err;
+    }
+  } else {
+    log_info(LD_REND, "Descriptor does not contain any introduction points.");
+    parsed->intro_nodes = smartlist_new();
+  }
+  /* We don't need the encoded/encrypted introduction points any longer. */
+  tor_free(intro_content);
+  /* Is descriptor too old? */
+  if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) {
+    log_warn(LD_REND, "Service descriptor with service ID %s is too old.",
+             safe_str_client(service_id));
+    goto err;
+  }
+  /* Is descriptor too far in the future? */
+  if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) {
+    log_warn(LD_REND, "Service descriptor with service ID %s is too far in "
+             "the future.", safe_str_client(service_id));
+    goto err;
+  }
+  /* Do we have the same exact copy already in our cache? */
+  tor_snprintf(key, sizeof(key), "2%s", service_id);
+  e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key);
+  if (e && !strcmp(desc, e->desc)) {
+    log_info(LD_REND,"We already have this service descriptor %s.",
+             safe_str_client(service_id));
+    goto okay;
+  }
+  /* Verify that we are not replacing an older descriptor. It's important to
+   * avoid an evil HSDir serving old descriptor. We validate if the
+   * timestamp is greater than and not equal because it's a rounded down
+   * timestamp to the hour so if the descriptor changed in the same hour,
+   * the rend cache failure will tell us if we have a new descriptor. */
+  if (e && e->parsed->timestamp > parsed->timestamp) {
+    log_info(LD_REND, "We already have a new enough service descriptor for "
+             "service ID %s with the same desc ID and version.",
+             safe_str_client(service_id));
+    goto okay;
+  }
+  /* Lookup our failure cache for intro point that might be unusable. */
+  validate_intro_point_failure(parsed, service_id);
+  /* It's now possible that our intro point list is empty, which means that
+   * this descriptor is useless to us because intro points have all failed
+   * somehow before. Discard the descriptor. */
+  if (smartlist_len(parsed->intro_nodes) == 0) {
+    log_info(LD_REND, "Service descriptor with service ID %s has no "
+             "usable intro points. Discarding it.",
+             safe_str_client(service_id));
+    goto err;
+  }
+  /* Now either purge the current one and replace its content or create a
+   * new one and add it to the rend cache. */
+  if (!e) {
+    e = tor_malloc_zero(sizeof(rend_cache_entry_t));
+    strmap_set_lc(rend_cache, key, e);
+  } else {
+    rend_cache_decrement_allocation(rend_cache_entry_allocation(e));
+    rend_cache_failure_remove(e->parsed);
+    rend_service_descriptor_free(e->parsed);
+    tor_free(e->desc);
+  }
+  e->parsed = parsed;
+  e->desc = tor_malloc_zero(encoded_size + 1);
+  strlcpy(e->desc, desc, encoded_size + 1);
+  e->len = encoded_size;
+  rend_cache_increment_allocation(rend_cache_entry_allocation(e));
+  log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.",
+            safe_str_client(service_id), (int)encoded_size);
+  if (entry) {
+    *entry = e;
+  }
+  return 0;
+
+ okay:
+  if (entry) {
+    *entry = e;
+  }
+  retval = 0;
+
+ err:
+  rend_service_descriptor_free(parsed);
+  tor_free(intro_content);
+  return retval;
+}
+
diff --git a/src/feature/rend/rendcache.h b/src/feature/rend/rendcache.h
new file mode 100644
index 0000000000..bb075409ec
--- /dev/null
+++ b/src/feature/rend/rendcache.h
@@ -0,0 +1,130 @@
+/* Copyright (c) 2015-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendcache.h
+ * \brief Header file for rendcache.c
+ **/
+
+#ifndef TOR_RENDCACHE_H
+#define TOR_RENDCACHE_H
+
+#include "or/or.h"
+#include "or/rendcommon.h"
+
+/** How old do we let hidden service descriptors get before discarding
+ * them as too old? */
+#define REND_CACHE_MAX_AGE (2*24*60*60)
+/** How wrong do we assume our clock may be when checking whether hidden
+ * services are too old or too new? */
+#define REND_CACHE_MAX_SKEW (24*60*60)
+/** How old do we keep an intro point failure entry in the failure cache? */
+#define REND_CACHE_FAILURE_MAX_AGE (5*60)
+
+/* Do not allow more than this many introduction points in a hidden service
+ * descriptor */
+#define MAX_INTRO_POINTS 10
+
+/** A cached rendezvous descriptor. */
+typedef struct rend_cache_entry_t {
+  size_t len; /**< Length of <b>desc</b> */
+  time_t last_served; /**< When did we last write this one to somebody?
+                       * (HSDir only) */
+  char *desc; /**< Service descriptor */
+  rend_service_descriptor_t *parsed; /**< Parsed value of 'desc' */
+} rend_cache_entry_t;
+
+/* Introduction point failure type. */
+typedef struct rend_cache_failure_intro_t {
+  /* When this intro point failure occurred thus we allocated this object and
+   * cache it. */
+  time_t created_ts;
+  rend_intro_point_failure_t failure_type;
+} rend_cache_failure_intro_t;
+
+/** Cache failure object indexed by service ID. */
+typedef struct rend_cache_failure_t {
+  /* Contains rend_cache_failure_intro_t indexed by identity digest. */
+  digestmap_t *intro_failures;
+} rend_cache_failure_t;
+
+typedef enum {
+  REND_CACHE_TYPE_CLIENT  = 1,
+  REND_CACHE_TYPE_SERVICE = 2,
+} rend_cache_type_t;
+
+/* Return maximum lifetime in seconds of a cache entry. */
+static inline time_t
+rend_cache_max_entry_lifetime(void)
+{
+  return REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW;
+}
+
+void rend_cache_init(void);
+void rend_cache_clean(time_t now, rend_cache_type_t cache_type);
+void rend_cache_failure_clean(time_t now);
+size_t rend_cache_clean_v2_descs_as_dir(time_t cutoff);
+void rend_cache_purge(void);
+void rend_cache_free_all(void);
+int rend_cache_lookup_entry(const char *query, int version,
+                            rend_cache_entry_t **entry_out);
+int rend_cache_lookup_v2_desc_as_service(const char *query,
+                                         rend_cache_entry_t **entry_out);
+int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc);
+
+int rend_cache_store_v2_desc_as_dir(const char *desc);
+int rend_cache_store_v2_desc_as_service(const char *desc);
+int rend_cache_store_v2_desc_as_client(const char *desc,
+                                       const char *desc_id_base32,
+                                       const rend_data_t *rend_query,
+                                       rend_cache_entry_t **entry);
+size_t rend_cache_get_total_allocation(void);
+
+void rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
+                                   const uint8_t *identity,
+                                   const char *service_id);
+void rend_cache_failure_purge(void);
+void rend_cache_decrement_allocation(size_t n);
+void rend_cache_increment_allocation(size_t n);
+
+#ifdef RENDCACHE_PRIVATE
+
+STATIC size_t rend_cache_entry_allocation(const rend_cache_entry_t *e);
+STATIC void rend_cache_entry_free_(rend_cache_entry_t *e);
+#define rend_cache_entry_free(e) \
+  FREE_AND_NULL(rend_cache_entry_t, rend_cache_entry_free_, (e))
+STATIC void rend_cache_failure_intro_entry_free_(rend_cache_failure_intro_t
+                                                 *entry);
+#define rend_cache_failure_intro_entry_free(e)                          \
+  FREE_AND_NULL(rend_cache_failure_intro_t,                   \
+                          rend_cache_failure_intro_entry_free_, (e))
+STATIC void rend_cache_failure_entry_free_(rend_cache_failure_t *entry);
+#define rend_cache_failure_entry_free(e)                        \
+  FREE_AND_NULL(rend_cache_failure_t,                 \
+                          rend_cache_failure_entry_free_, (e))
+STATIC int cache_failure_intro_lookup(const uint8_t *identity,
+                                      const char *service_id,
+                                      rend_cache_failure_intro_t
+                                      **intro_entry);
+STATIC rend_cache_failure_intro_t *rend_cache_failure_intro_entry_new(
+                                      rend_intro_point_failure_t failure);
+STATIC rend_cache_failure_t *rend_cache_failure_entry_new(void);
+STATIC void rend_cache_failure_remove(rend_service_descriptor_t *desc);
+STATIC void cache_failure_intro_add(const uint8_t *identity,
+                                    const char *service_id,
+                                    rend_intro_point_failure_t failure);
+STATIC void validate_intro_point_failure(const rend_service_descriptor_t *desc,
+                                        const char *service_id);
+
+STATIC void rend_cache_failure_entry_free_void(void *entry);
+
+#ifdef TOR_UNIT_TESTS
+extern strmap_t *rend_cache;
+extern strmap_t *rend_cache_failure;
+extern digestmap_t *rend_cache_v2_dir;
+extern size_t rend_cache_total_allocation;
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(RENDCACHE_PRIVATE) */
+
+#endif /* !defined(TOR_RENDCACHE_H) */
+
diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c
new file mode 100644
index 0000000000..d4262f2f38
--- /dev/null
+++ b/src/feature/rend/rendclient.c
@@ -0,0 +1,1257 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendclient.c
+ * \brief Client code to access location-hidden services.
+ **/
+
+#include "or/or.h"
+#include "or/circpathbias.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "or/connection.h"
+#include "or/connection_edge.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/directory.h"
+#include "or/hs_circuit.h"
+#include "or/hs_client.h"
+#include "or/hs_common.h"
+#include "or/main.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/relay.h"
+#include "or/rendclient.h"
+#include "or/rendcommon.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerset.h"
+#include "lib/encoding/confline.h"
+
+#include "or/cpath_build_state_st.h"
+#include "or/crypt_path_st.h"
+#include "or/dir_connection_st.h"
+#include "or/entry_connection_st.h"
+#include "or/extend_info_st.h"
+#include "or/origin_circuit_st.h"
+#include "or/rend_intro_point_st.h"
+#include "or/rend_service_descriptor_st.h"
+#include "or/routerstatus_st.h"
+
+static extend_info_t *rend_client_get_random_intro_impl(
+                          const rend_cache_entry_t *rend_query,
+                          const int strict, const int warnings);
+
+/** Purge all potentially remotely-detectable state held in the hidden
+ * service client code.  Called on SIGNAL NEWNYM. */
+void
+rend_client_purge_state(void)
+{
+  rend_cache_purge();
+  rend_cache_failure_purge();
+  rend_client_cancel_descriptor_fetches();
+  hs_purge_last_hid_serv_requests();
+}
+
+/** Called when we've established a circuit to an introduction point:
+ * send the introduction request. */
+void
+rend_client_introcirc_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
+  tor_assert(circ->cpath);
+
+  log_info(LD_REND,"introcirc is open");
+  connection_ap_attach_pending(1);
+}
+
+/** Send the establish-rendezvous cell along a rendezvous circuit. if
+ * it fails, mark the circ for close and return -1. else return 0.
+ */
+static int
+rend_client_send_establish_rendezvous(origin_circuit_t *circ)
+{
+  tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+  tor_assert(circ->rend_data);
+  log_info(LD_REND, "Sending an ESTABLISH_RENDEZVOUS cell");
+
+  crypto_rand(circ->rend_data->rend_cookie, REND_COOKIE_LEN);
+
+  /* Set timestamp_dirty, because circuit_expire_building expects it,
+   * and the rend cookie also means we've used the circ. */
+  circ->base_.timestamp_dirty = time(NULL);
+
+  /* We've attempted to use this circuit. Probe it if we fail */
+  pathbias_count_use_attempt(circ);
+
+  if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_ESTABLISH_RENDEZVOUS,
+                                   circ->rend_data->rend_cookie,
+                                   REND_COOKIE_LEN,
+                                   circ->cpath->prev)<0) {
+    /* circ is already marked for close */
+    log_warn(LD_GENERAL, "Couldn't send ESTABLISH_RENDEZVOUS cell");
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Called when we're trying to connect an ap conn; sends an INTRODUCE1 cell
+ * down introcirc if possible.
+ */
+int
+rend_client_send_introduction(origin_circuit_t *introcirc,
+                              origin_circuit_t *rendcirc)
+{
+  const or_options_t *options = get_options();
+  size_t payload_len;
+  int r, v3_shift = 0;
+  char payload[RELAY_PAYLOAD_SIZE];
+  char tmp[RELAY_PAYLOAD_SIZE];
+  rend_cache_entry_t *entry = NULL;
+  crypt_path_t *cpath;
+  off_t dh_offset;
+  crypto_pk_t *intro_key = NULL;
+  int status = 0;
+  const char *onion_address;
+
+  tor_assert(introcirc->base_.purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
+  tor_assert(rendcirc->base_.purpose == CIRCUIT_PURPOSE_C_REND_READY);
+  tor_assert(introcirc->rend_data);
+  tor_assert(rendcirc->rend_data);
+  tor_assert(!rend_cmp_service_ids(rend_data_get_address(introcirc->rend_data),
+                                  rend_data_get_address(rendcirc->rend_data)));
+  assert_circ_anonymity_ok(introcirc, options);
+  assert_circ_anonymity_ok(rendcirc, options);
+  onion_address = rend_data_get_address(introcirc->rend_data);
+
+  r = rend_cache_lookup_entry(onion_address, -1, &entry);
+  /* An invalid onion address is not possible else we have a big issue. */
+  tor_assert(r != -EINVAL);
+  if (r < 0 || !rend_client_any_intro_points_usable(entry)) {
+    /* If the descriptor is not found or the intro points are not usable
+     * anymore, trigger a fetch. */
+    log_info(LD_REND,
+             "query %s didn't have valid rend desc in cache. "
+             "Refetching descriptor.",
+             safe_str_client(onion_address));
+    rend_client_refetch_v2_renddesc(introcirc->rend_data);
+    {
+      connection_t *conn;
+
+      while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP,
+                       AP_CONN_STATE_CIRCUIT_WAIT, onion_address))) {
+        connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
+        conn->state = AP_CONN_STATE_RENDDESC_WAIT;
+      }
+    }
+
+    status = -1;
+    goto cleanup;
+  }
+
+  /* first 20 bytes of payload are the hash of the service's pk */
+  intro_key = NULL;
+  SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *,
+                    intro, {
+    if (tor_memeq(introcirc->build_state->chosen_exit->identity_digest,
+                intro->extend_info->identity_digest, DIGEST_LEN)) {
+      intro_key = intro->intro_key;
+      break;
+    }
+  });
+  if (!intro_key) {
+    log_info(LD_REND, "Could not find intro key for %s at %s; we "
+             "have a v2 rend desc with %d intro points. "
+             "Trying a different intro point...",
+             safe_str_client(onion_address),
+             safe_str_client(extend_info_describe(
+                                   introcirc->build_state->chosen_exit)),
+             smartlist_len(entry->parsed->intro_nodes));
+
+    if (hs_client_reextend_intro_circuit(introcirc)) {
+      status = -2;
+      goto perm_err;
+    } else {
+      status = -1;
+      goto cleanup;
+    }
+  }
+  if (crypto_pk_get_digest(intro_key, payload)<0) {
+    log_warn(LD_BUG, "Internal error: couldn't hash public key.");
+    status = -2;
+    goto perm_err;
+  }
+
+  /* Initialize the pending_final_cpath and start the DH handshake. */
+  cpath = rendcirc->build_state->pending_final_cpath;
+  if (!cpath) {
+    cpath = rendcirc->build_state->pending_final_cpath =
+      tor_malloc_zero(sizeof(crypt_path_t));
+    cpath->magic = CRYPT_PATH_MAGIC;
+    if (!(cpath->rend_dh_handshake_state = crypto_dh_new(DH_TYPE_REND))) {
+      log_warn(LD_BUG, "Internal error: couldn't allocate DH.");
+      status = -2;
+      goto perm_err;
+    }
+    if (crypto_dh_generate_public(cpath->rend_dh_handshake_state)<0) {
+      log_warn(LD_BUG, "Internal error: couldn't generate g^x.");
+      status = -2;
+      goto perm_err;
+    }
+  }
+
+  /* If version is 3, write (optional) auth data and timestamp. */
+  if (entry->parsed->protocols & (1<<3)) {
+    tmp[0] = 3; /* version 3 of the cell format */
+    /* auth type, if any */
+    tmp[1] = (uint8_t) TO_REND_DATA_V2(introcirc->rend_data)->auth_type;
+    v3_shift = 1;
+    if (tmp[1] != REND_NO_AUTH) {
+      set_uint16(tmp+2, htons(REND_DESC_COOKIE_LEN));
+      memcpy(tmp+4, TO_REND_DATA_V2(introcirc->rend_data)->descriptor_cookie,
+             REND_DESC_COOKIE_LEN);
+      v3_shift += 2+REND_DESC_COOKIE_LEN;
+    }
+    /* Once this held a timestamp. */
+    set_uint32(tmp+v3_shift+1, 0);
+    v3_shift += 4;
+  } /* if version 2 only write version number */
+  else if (entry->parsed->protocols & (1<<2)) {
+    tmp[0] = 2; /* version 2 of the cell format */
+  }
+
+  /* write the remaining items into tmp */
+  if (entry->parsed->protocols & (1<<3) || entry->parsed->protocols & (1<<2)) {
+    /* version 2 format */
+    extend_info_t *extend_info = rendcirc->build_state->chosen_exit;
+    int klen;
+    /* nul pads */
+    set_uint32(tmp+v3_shift+1, tor_addr_to_ipv4n(&extend_info->addr));
+    set_uint16(tmp+v3_shift+5, htons(extend_info->port));
+    memcpy(tmp+v3_shift+7, extend_info->identity_digest, DIGEST_LEN);
+    klen = crypto_pk_asn1_encode(extend_info->onion_key,
+                                 tmp+v3_shift+7+DIGEST_LEN+2,
+                                 sizeof(tmp)-(v3_shift+7+DIGEST_LEN+2));
+    if (klen < 0) {
+      log_warn(LD_BUG,"Internal error: can't encode public key.");
+      status = -2;
+      goto perm_err;
+    }
+    set_uint16(tmp+v3_shift+7+DIGEST_LEN, htons(klen));
+    memcpy(tmp+v3_shift+7+DIGEST_LEN+2+klen, rendcirc->rend_data->rend_cookie,
+           REND_COOKIE_LEN);
+    dh_offset = v3_shift+7+DIGEST_LEN+2+klen+REND_COOKIE_LEN;
+  } else {
+    /* Version 0. */
+    strncpy(tmp, rendcirc->build_state->chosen_exit->nickname,
+            (MAX_NICKNAME_LEN+1)); /* nul pads */
+    memcpy(tmp+MAX_NICKNAME_LEN+1, rendcirc->rend_data->rend_cookie,
+           REND_COOKIE_LEN);
+    dh_offset = MAX_NICKNAME_LEN+1+REND_COOKIE_LEN;
+  }
+
+  if (crypto_dh_get_public(cpath->rend_dh_handshake_state, tmp+dh_offset,
+                           DH1024_KEY_LEN)<0) {
+    log_warn(LD_BUG, "Internal error: couldn't extract g^x.");
+    status = -2;
+    goto perm_err;
+  }
+
+  /*XXX maybe give crypto_pk_obsolete_public_hybrid_encrypt a max_len arg,
+   * to avoid buffer overflows? */
+  r = crypto_pk_obsolete_public_hybrid_encrypt(intro_key, payload+DIGEST_LEN,
+                                      sizeof(payload)-DIGEST_LEN,
+                                      tmp,
+                                      (int)(dh_offset+DH1024_KEY_LEN),
+                                      PK_PKCS1_OAEP_PADDING, 0);
+  if (r<0) {
+    log_warn(LD_BUG,"Internal error: hybrid pk encrypt failed.");
+    status = -2;
+    goto perm_err;
+  }
+
+  payload_len = DIGEST_LEN + r;
+  tor_assert(payload_len <= RELAY_PAYLOAD_SIZE); /* we overran something */
+
+  /* Copy the rendezvous cookie from rendcirc to introcirc, so that
+   * when introcirc gets an ack, we can change the state of the right
+   * rendezvous circuit. */
+  memcpy(introcirc->rend_data->rend_cookie, rendcirc->rend_data->rend_cookie,
+         REND_COOKIE_LEN);
+
+  log_info(LD_REND, "Sending an INTRODUCE1 cell");
+  if (relay_send_command_from_edge(0, TO_CIRCUIT(introcirc),
+                                   RELAY_COMMAND_INTRODUCE1,
+                                   payload, payload_len,
+                                   introcirc->cpath->prev)<0) {
+    /* introcirc is already marked for close. leave rendcirc alone. */
+    log_warn(LD_BUG, "Couldn't send INTRODUCE1 cell");
+    status = -2;
+    goto cleanup;
+  }
+
+  /* Now, we wait for an ACK or NAK on this circuit. */
+  circuit_change_purpose(TO_CIRCUIT(introcirc),
+                         CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT);
+  /* Set timestamp_dirty, because circuit_expire_building expects it
+   * to specify when a circuit entered the _C_INTRODUCE_ACK_WAIT
+   * state. */
+  introcirc->base_.timestamp_dirty = time(NULL);
+
+  pathbias_count_use_attempt(introcirc);
+
+  goto cleanup;
+
+ perm_err:
+  if (!introcirc->base_.marked_for_close)
+    circuit_mark_for_close(TO_CIRCUIT(introcirc), END_CIRC_REASON_INTERNAL);
+  circuit_mark_for_close(TO_CIRCUIT(rendcirc), END_CIRC_REASON_INTERNAL);
+ cleanup:
+  memwipe(payload, 0, sizeof(payload));
+  memwipe(tmp, 0, sizeof(tmp));
+
+  return status;
+}
+
+/** Called when a rendezvous circuit is open; sends a establish
+ * rendezvous circuit as appropriate. */
+void
+rend_client_rendcirc_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+
+  log_info(LD_REND,"rendcirc is open");
+
+  /* generate a rendezvous cookie, store it in circ */
+  if (rend_client_send_establish_rendezvous(circ) < 0) {
+    return;
+  }
+}
+
+/**
+ * Called to close other intro circuits we launched in parallel.
+ */
+static void
+rend_client_close_other_intros(const uint8_t *rend_pk_digest)
+{
+  /* abort parallel intro circs, if any */
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, c) {
+    if ((c->purpose == CIRCUIT_PURPOSE_C_INTRODUCING ||
+        c->purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) &&
+        !c->marked_for_close && CIRCUIT_IS_ORIGIN(c)) {
+      origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(c);
+      if (oc->rend_data &&
+          rend_circuit_pk_digest_eq(oc, rend_pk_digest)) {
+        log_info(LD_REND|LD_CIRC, "Closing introduction circuit %d that we "
+                 "built in parallel (Purpose %d).", oc->global_identifier,
+                 c->purpose);
+        circuit_mark_for_close(c, END_CIRC_REASON_IP_NOW_REDUNDANT);
+      }
+    }
+  }
+  SMARTLIST_FOREACH_END(c);
+}
+
+/** Called when get an ACK or a NAK for a REND_INTRODUCE1 cell.
+ */
+int
+rend_client_introduction_acked(origin_circuit_t *circ,
+                               const uint8_t *request, size_t request_len)
+{
+  const or_options_t *options = get_options();
+  origin_circuit_t *rendcirc;
+  (void) request; // XXXX Use this.
+
+  tor_assert(circ->build_state);
+  tor_assert(circ->build_state->chosen_exit);
+  assert_circ_anonymity_ok(circ, options);
+  tor_assert(circ->rend_data);
+
+  if (request_len == 0) {
+    /* It's an ACK; the introduction point relayed our introduction request. */
+    /* Locate the rend circ which is waiting to hear about this ack,
+     * and tell it.
+     */
+    log_info(LD_REND,"Received ack. Telling rend circ...");
+    rendcirc = circuit_get_ready_rend_circ_by_rend_data(circ->rend_data);
+    if (rendcirc) { /* remember the ack */
+      assert_circ_anonymity_ok(rendcirc, options);
+      circuit_change_purpose(TO_CIRCUIT(rendcirc),
+                             CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+      /* Set timestamp_dirty, because circuit_expire_building expects
+       * it to specify when a circuit entered the
+       * _C_REND_READY_INTRO_ACKED state. */
+      rendcirc->base_.timestamp_dirty = time(NULL);
+    } else {
+      log_info(LD_REND,"...Found no rend circ. Dropping on the floor.");
+    }
+    /* close the circuit: we won't need it anymore. */
+    circuit_change_purpose(TO_CIRCUIT(circ),
+                           CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
+
+    /* close any other intros launched in parallel */
+    rend_client_close_other_intros(rend_data_get_pk_digest(circ->rend_data,
+                                                           NULL));
+  } else {
+    /* It's a NAK; the introduction point didn't relay our request. */
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
+    /* Remove this intro point from the set of viable introduction
+     * points. If any remain, extend to a new one and try again.
+     * If none remain, refetch the service descriptor.
+     */
+    log_info(LD_REND, "Got nack for %s from %s...",
+        safe_str_client(rend_data_get_address(circ->rend_data)),
+        safe_str_client(extend_info_describe(circ->build_state->chosen_exit)));
+    if (rend_client_report_intro_point_failure(circ->build_state->chosen_exit,
+                                             circ->rend_data,
+                                             INTRO_POINT_FAILURE_GENERIC)>0) {
+      /* There are introduction points left. Re-extend the circuit to
+       * another intro point and try again. */
+      int result = hs_client_reextend_intro_circuit(circ);
+      /* XXXX If that call failed, should we close the rend circuit,
+       * too? */
+      return result;
+    } else {
+      /* Close circuit because no more intro points are usable thus not
+       * useful anymore. Change it's purpose before so we don't report an
+       * intro point failure again triggering an extra descriptor fetch. */
+      circuit_change_purpose(TO_CIRCUIT(circ),
+          CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
+      circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
+    }
+  }
+  return 0;
+}
+
+/** Determine the responsible hidden service directories for <b>desc_id</b>
+ * and fetch the descriptor with that ID from one of them. Only
+ * send a request to a hidden service directory that we have not yet tried
+ * during this attempt to connect to this hidden service; on success, return 1,
+ * in the case that no hidden service directory is left to ask for the
+ * descriptor, return 0, and in case of a failure -1.  */
+static int
+directory_get_from_hs_dir(const char *desc_id,
+                          const rend_data_t *rend_query,
+                          routerstatus_t *rs_hsdir)
+{
+  routerstatus_t *hs_dir = rs_hsdir;
+  char *hsdir_fp;
+  char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+  char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64];
+  const rend_data_v2_t *rend_data;
+#ifdef ENABLE_TOR2WEB_MODE
+  const int tor2web_mode = get_options()->Tor2webMode;
+  const int how_to_fetch = tor2web_mode ? DIRIND_ONEHOP : DIRIND_ANONYMOUS;
+#else
+  const int how_to_fetch = DIRIND_ANONYMOUS;
+#endif /* defined(ENABLE_TOR2WEB_MODE) */
+
+  tor_assert(desc_id);
+  tor_assert(rend_query);
+  rend_data = TO_REND_DATA_V2(rend_query);
+
+  base32_encode(desc_id_base32, sizeof(desc_id_base32),
+                desc_id, DIGEST_LEN);
+
+  /* Automatically pick an hs dir if none given. */
+  if (!rs_hsdir) {
+    /* Determine responsible dirs. Even if we can't get all we want, work with
+     * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */
+    smartlist_t *responsible_dirs = smartlist_new();
+    hid_serv_get_responsible_directories(responsible_dirs, desc_id);
+
+    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32);
+    if (!hs_dir) {
+      /* No suitable hs dir can be found, stop right now. */
+      control_event_hsv2_descriptor_failed(rend_query, NULL,
+                                           "QUERY_NO_HSDIR");
+      control_event_hs_descriptor_content(rend_data_get_address(rend_query),
+                                          desc_id_base32, NULL, NULL);
+      return 0;
+    }
+  }
+
+  /* Add a copy of the HSDir identity digest to the query so we can track it
+   * on the control port. */
+  hsdir_fp = tor_memdup(hs_dir->identity_digest,
+                        sizeof(hs_dir->identity_digest));
+  smartlist_add(rend_query->hsdirs_fp, hsdir_fp);
+
+  /* Encode descriptor cookie for logging purposes. Also, if the cookie is
+   * malformed, no fetch is triggered thus this needs to be done before the
+   * fetch request. */
+  if (rend_data->auth_type != REND_NO_AUTH) {
+    if (base64_encode(descriptor_cookie_base64,
+                      sizeof(descriptor_cookie_base64),
+                      rend_data->descriptor_cookie,
+                      REND_DESC_COOKIE_LEN,
+                      0)<0) {
+      log_warn(LD_BUG, "Could not base64-encode descriptor cookie.");
+      control_event_hsv2_descriptor_failed(rend_query, hsdir_fp, "BAD_DESC");
+      control_event_hs_descriptor_content(rend_data_get_address(rend_query),
+                                          desc_id_base32, hsdir_fp, NULL);
+      return 0;
+    }
+    /* Remove == signs. */
+    descriptor_cookie_base64[strlen(descriptor_cookie_base64)-2] = '\0';
+  } else {
+    strlcpy(descriptor_cookie_base64, "(none)",
+            sizeof(descriptor_cookie_base64));
+  }
+
+  /* Send fetch request. (Pass query and possibly descriptor cookie so that
+   * they can be written to the directory connection and be referred to when
+   * the response arrives. */
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_RENDDESC_V2);
+  directory_request_set_routerstatus(req, hs_dir);
+  directory_request_set_indirection(req, how_to_fetch);
+  directory_request_set_resource(req, desc_id_base32);
+  directory_request_set_rend_query(req, rend_query);
+  directory_initiate_request(req);
+  directory_request_free(req);
+
+  log_info(LD_REND, "Sending fetch request for v2 descriptor for "
+                    "service '%s' with descriptor ID '%s', auth type %d, "
+                    "and descriptor cookie '%s' to hidden service "
+                    "directory %s",
+           rend_data->onion_address, desc_id_base32,
+           rend_data->auth_type,
+           (rend_data->auth_type == REND_NO_AUTH ? "[none]" :
+            escaped_safe_str_client(descriptor_cookie_base64)),
+           routerstatus_describe(hs_dir));
+  control_event_hs_descriptor_requested(rend_data->onion_address,
+                                        rend_data->auth_type,
+                                        hs_dir->identity_digest,
+                                        desc_id_base32, NULL);
+  return 1;
+}
+
+/** Remove tracked HSDir requests from our history for this hidden service
+ *  descriptor <b>desc_id</b> (of size DIGEST_LEN) */
+static void
+purge_v2_hidserv_req(const char *desc_id)
+{
+  char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+
+  /* The hsdir request tracker stores v2 keys using the base32 encoded
+     desc_id. Do it: */
+  base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                DIGEST_LEN);
+  hs_purge_hid_serv_from_last_hid_serv_requests(desc_id_base32);
+}
+
+/** Fetch a v2 descriptor using the given descriptor id. If any hsdir(s) are
+ * given, they will be used instead.
+ *
+ * On success, 1 is returned. If no hidden service is left to ask, return 0.
+ * On error, -1 is returned. */
+static int
+fetch_v2_desc_by_descid(const char *desc_id,
+                        const rend_data_t *rend_query, smartlist_t *hsdirs)
+{
+  int ret;
+
+  tor_assert(rend_query);
+
+  if (!hsdirs) {
+    ret = directory_get_from_hs_dir(desc_id, rend_query, NULL);
+    goto end; /* either success or failure, but we're done */
+  }
+
+  /* Using the given hsdir list, trigger a fetch on each of them. */
+  SMARTLIST_FOREACH_BEGIN(hsdirs, routerstatus_t *, hs_dir) {
+    /* This should always be a success. */
+    ret = directory_get_from_hs_dir(desc_id, rend_query, hs_dir);
+    tor_assert(ret);
+  } SMARTLIST_FOREACH_END(hs_dir);
+
+  /* Everything went well. */
+  ret = 0;
+
+ end:
+  return ret;
+}
+
+/** Fetch a v2 descriptor using the onion address in the given query object.
+ * This will compute the descriptor id for each replicas and fetch it on the
+ * given hsdir(s) if any or the responsible ones that are chosen
+ * automatically.
+ *
+ * On success, 1 is returned. If no hidden service is left to ask, return 0.
+ * On error, -1 is returned. */
+static int
+fetch_v2_desc_by_addr(rend_data_t *rend_query, smartlist_t *hsdirs)
+{
+  char descriptor_id[DIGEST_LEN];
+  int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS];
+  int i, tries_left, ret;
+  rend_data_v2_t *rend_data = TO_REND_DATA_V2(rend_query);
+
+  /* Randomly iterate over the replicas until a descriptor can be fetched
+   * from one of the consecutive nodes, or no options are left. */
+  for (i = 0; i < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; i++) {
+    replicas_left_to_try[i] = i;
+  }
+
+  tries_left = REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
+  while (tries_left > 0) {
+    int rand_val = crypto_rand_int(tries_left);
+    int chosen_replica = replicas_left_to_try[rand_val];
+    replicas_left_to_try[rand_val] = replicas_left_to_try[--tries_left];
+
+    ret = rend_compute_v2_desc_id(descriptor_id,
+                                  rend_data->onion_address,
+                                  rend_data->auth_type == REND_STEALTH_AUTH ?
+                                    rend_data->descriptor_cookie : NULL,
+                                  time(NULL), chosen_replica);
+    if (ret < 0) {
+      /* Normally, on failure the descriptor_id is untouched but let's be
+       * safe in general in case the function changes at some point. */
+      goto end;
+    }
+
+    if (tor_memcmp(descriptor_id, rend_data->descriptor_id[chosen_replica],
+                   sizeof(descriptor_id)) != 0) {
+      /* Not equal from what we currently have so purge the last hid serv
+       * request cache and update the descriptor ID with the new value. */
+      purge_v2_hidserv_req(rend_data->descriptor_id[chosen_replica]);
+      memcpy(rend_data->descriptor_id[chosen_replica], descriptor_id,
+             sizeof(rend_data->descriptor_id[chosen_replica]));
+    }
+
+    /* Trigger the fetch with the computed descriptor ID. */
+    ret = fetch_v2_desc_by_descid(descriptor_id, rend_query, hsdirs);
+    if (ret != 0) {
+      /* Either on success or failure, as long as we tried a fetch we are
+       * done here. */
+      goto end;
+    }
+  }
+
+  /* If we come here, there are no hidden service directories left. */
+  log_info(LD_REND, "Could not pick one of the responsible hidden "
+                    "service directories to fetch descriptors, because "
+                    "we already tried them all unsuccessfully.");
+  ret = 0;
+
+ end:
+  memwipe(descriptor_id, 0, sizeof(descriptor_id));
+  return ret;
+}
+
+/** Fetch a v2 descriptor using the given query. If any hsdir are specified,
+ * use them for the fetch.
+ *
+ * On success, 1 is returned. If no hidden service is left to ask, return 0.
+ * On error, -1 is returned. */
+int
+rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs)
+{
+  int ret;
+  rend_data_v2_t *rend_data;
+  const char *onion_address;
+
+  tor_assert(query);
+
+  /* Get the version 2 data structure of the query. */
+  rend_data = TO_REND_DATA_V2(query);
+  onion_address = rend_data_get_address(query);
+
+  /* Depending on what's available in the rend data query object, we will
+   * trigger a fetch by HS address or using a descriptor ID. */
+
+  if (onion_address[0] != '\0') {
+    ret = fetch_v2_desc_by_addr(query, hsdirs);
+  } else if (!tor_digest_is_zero(rend_data->desc_id_fetch)) {
+    ret = fetch_v2_desc_by_descid(rend_data->desc_id_fetch, query,
+                                  hsdirs);
+  } else {
+    /* Query data is invalid. */
+    ret = -1;
+    goto error;
+  }
+
+ error:
+  return ret;
+}
+
+/** Unless we already have a descriptor for <b>rend_query</b> with at least
+ * one (possibly) working introduction point in it, start a connection to a
+ * hidden service directory to fetch a v2 rendezvous service descriptor. */
+void
+rend_client_refetch_v2_renddesc(rend_data_t *rend_query)
+{
+  rend_cache_entry_t *e = NULL;
+  const char *onion_address = rend_data_get_address(rend_query);
+
+  tor_assert(rend_query);
+  /* Before fetching, check if we already have a usable descriptor here. */
+  if (rend_cache_lookup_entry(onion_address, -1, &e) == 0 &&
+      rend_client_any_intro_points_usable(e)) {
+    log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we "
+                      "already have a usable descriptor here. Not fetching.");
+    return;
+  }
+  /* Are we configured to fetch descriptors? */
+  if (!get_options()->FetchHidServDescriptors) {
+    log_warn(LD_REND, "We received an onion address for a v2 rendezvous "
+        "service descriptor, but are not fetching service descriptors.");
+    return;
+  }
+  log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s",
+            safe_str_client(onion_address));
+
+  rend_client_fetch_v2_desc(rend_query, NULL);
+  /* We don't need to look the error code because either on failure or
+   * success, the necessary steps to continue the HS connection will be
+   * triggered once the descriptor arrives or if all fetch failed. */
+  return;
+}
+
+/** Cancel all rendezvous descriptor fetches currently in progress.
+ */
+void
+rend_client_cancel_descriptor_fetches(void)
+{
+  smartlist_t *connection_array = get_connection_array();
+
+  SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_DIR &&
+        conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) {
+      /* It's a rendezvous descriptor fetch in progress -- cancel it
+       * by marking the connection for close.
+       *
+       * Even if this connection has already reached EOF, this is
+       * enough to make sure that if the descriptor hasn't been
+       * processed yet, it won't be.  See the end of
+       * connection_handle_read; connection_reached_eof (indirectly)
+       * processes whatever response the connection received. */
+
+      const rend_data_t *rd = (TO_DIR_CONN(conn))->rend_data;
+      if (!rd) {
+        log_warn(LD_BUG | LD_REND,
+                 "Marking for close dir conn fetching rendezvous "
+                 "descriptor for unknown service!");
+      } else {
+        log_debug(LD_REND, "Marking for close dir conn fetching "
+                  "rendezvous descriptor for service %s",
+                  safe_str(rend_data_get_address(rd)));
+      }
+      connection_mark_for_close(conn);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+}
+
+/** Mark <b>failed_intro</b> as a failed introduction point for the
+ * hidden service specified by <b>rend_query</b>. If the HS now has no
+ * usable intro points, or we do not have an HS descriptor for it,
+ * then launch a new renddesc fetch.
+ *
+ * If <b>failure_type</b> is INTRO_POINT_FAILURE_GENERIC, remove the
+ * intro point from (our parsed copy of) the HS descriptor.
+ *
+ * If <b>failure_type</b> is INTRO_POINT_FAILURE_TIMEOUT, mark the
+ * intro point as 'timed out'; it will not be retried until the
+ * current hidden service connection attempt has ended or it has
+ * appeared in a newly fetched rendezvous descriptor.
+ *
+ * If <b>failure_type</b> is INTRO_POINT_FAILURE_UNREACHABLE,
+ * increment the intro point's reachability-failure count; if it has
+ * now failed MAX_INTRO_POINT_REACHABILITY_FAILURES or more times,
+ * remove the intro point from (our parsed copy of) the HS descriptor.
+ *
+ * Return -1 if error, 0 if no usable intro points remain or service
+ * unrecognized, 1 if recognized and some intro points remain.
+ */
+int
+rend_client_report_intro_point_failure(extend_info_t *failed_intro,
+                                       rend_data_t *rend_data,
+                                       unsigned int failure_type)
+{
+  int i, r;
+  rend_cache_entry_t *ent;
+  connection_t *conn;
+  const char *onion_address = rend_data_get_address(rend_data);
+
+  r = rend_cache_lookup_entry(onion_address, -1, &ent);
+  if (r < 0) {
+    /* Either invalid onion address or cache entry not found. */
+    switch (-r) {
+    case EINVAL:
+      log_warn(LD_BUG, "Malformed service ID %s.",
+               escaped_safe_str_client(onion_address));
+      return -1;
+    case ENOENT:
+      log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.",
+               escaped_safe_str_client(onion_address));
+      rend_client_refetch_v2_renddesc(rend_data);
+      return 0;
+    default:
+      log_warn(LD_BUG, "Unknown cache lookup returned code: %d", r);
+      return -1;
+    }
+  }
+  /* The intro points are not checked here if they are usable or not because
+   * this is called when an intro point circuit is closed thus there must be
+   * at least one intro point that is usable and is about to be flagged. */
+
+  for (i = 0; i < smartlist_len(ent->parsed->intro_nodes); i++) {
+    rend_intro_point_t *intro = smartlist_get(ent->parsed->intro_nodes, i);
+    if (tor_memeq(failed_intro->identity_digest,
+                intro->extend_info->identity_digest, DIGEST_LEN)) {
+      switch (failure_type) {
+      default:
+        log_warn(LD_BUG, "Unknown failure type %u. Removing intro point.",
+                 failure_type);
+        tor_fragile_assert();
+        /* fall through */
+      case INTRO_POINT_FAILURE_GENERIC:
+        rend_cache_intro_failure_note(failure_type,
+                                      (uint8_t *)failed_intro->identity_digest,
+                                      onion_address);
+        rend_intro_point_free(intro);
+        smartlist_del(ent->parsed->intro_nodes, i);
+        break;
+      case INTRO_POINT_FAILURE_TIMEOUT:
+        intro->timed_out = 1;
+        break;
+      case INTRO_POINT_FAILURE_UNREACHABLE:
+        ++(intro->unreachable_count);
+        {
+          int zap_intro_point =
+            intro->unreachable_count >= MAX_INTRO_POINT_REACHABILITY_FAILURES;
+          log_info(LD_REND, "Failed to reach this intro point %u times.%s",
+                   intro->unreachable_count,
+                   zap_intro_point ? " Removing from descriptor.": "");
+          if (zap_intro_point) {
+            rend_cache_intro_failure_note(
+                failure_type,
+                (uint8_t *) failed_intro->identity_digest, onion_address);
+            rend_intro_point_free(intro);
+            smartlist_del(ent->parsed->intro_nodes, i);
+          }
+        }
+        break;
+      }
+      break;
+    }
+  }
+
+  if (! rend_client_any_intro_points_usable(ent)) {
+    log_info(LD_REND,
+             "No more intro points remain for %s. Re-fetching descriptor.",
+             escaped_safe_str_client(onion_address));
+    rend_client_refetch_v2_renddesc(rend_data);
+
+    /* move all pending streams back to renddesc_wait */
+    /* NOTE: We can now do this faster, if we use pending_entry_connections */
+    while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP,
+                                   AP_CONN_STATE_CIRCUIT_WAIT,
+                                   onion_address))) {
+      connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
+      conn->state = AP_CONN_STATE_RENDDESC_WAIT;
+    }
+
+    return 0;
+  }
+  log_info(LD_REND,"%d options left for %s.",
+           smartlist_len(ent->parsed->intro_nodes),
+           escaped_safe_str_client(onion_address));
+  return 1;
+}
+
+/** The service sent us a rendezvous cell; join the circuits. */
+int
+rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request,
+                               size_t request_len)
+{
+  if (request_len != DH1024_KEY_LEN+DIGEST_LEN) {
+    log_warn(LD_PROTOCOL,"Incorrect length (%d) on RENDEZVOUS2 cell.",
+             (int)request_len);
+    goto err;
+  }
+
+  if (hs_circuit_setup_e2e_rend_circ_legacy_client(circ, request) < 0) {
+    log_warn(LD_GENERAL, "Failed to setup circ");
+    goto err;
+  }
+  return 0;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that are
+ * waiting on <b>query</b>. If there's a working cache entry here with at
+ * least one intro point, move them to the next state. */
+void
+rend_client_desc_trynow(const char *query)
+{
+  entry_connection_t *conn;
+  rend_cache_entry_t *entry;
+  const rend_data_t *rend_data;
+  time_t now = time(NULL);
+
+  smartlist_t *conns = get_connection_array();
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    if (base_conn->type != CONN_TYPE_AP ||
+        base_conn->state != AP_CONN_STATE_RENDDESC_WAIT ||
+        base_conn->marked_for_close)
+      continue;
+    conn = TO_ENTRY_CONN(base_conn);
+    rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data;
+    if (!rend_data)
+      continue;
+    const char *onion_address = rend_data_get_address(rend_data);
+    if (rend_cmp_service_ids(query, onion_address))
+      continue;
+    assert_connection_ok(base_conn, now);
+    if (rend_cache_lookup_entry(onion_address, -1,
+                                &entry) == 0 &&
+        rend_client_any_intro_points_usable(entry)) {
+      /* either this fetch worked, or it failed but there was a
+       * valid entry from before which we should reuse */
+      log_info(LD_REND,"Rend desc is usable. Launching circuits.");
+      base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT;
+
+      /* restart their timeout values, so they get a fair shake at
+       * connecting to the hidden service. */
+      base_conn->timestamp_created = now;
+      base_conn->timestamp_last_read_allowed = now;
+      base_conn->timestamp_last_write_allowed = now;
+
+      connection_ap_mark_as_pending_circuit(conn);
+    } else { /* 404, or fetch didn't get that far */
+      log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is "
+                 "unavailable (try again later).",
+                 safe_str_client(query));
+      connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED);
+      rend_client_note_connection_attempt_ended(rend_data);
+    }
+  } SMARTLIST_FOREACH_END(base_conn);
+}
+
+/** Clear temporary state used only during an attempt to connect to the
+ * hidden service with <b>rend_data</b>. Called when a connection attempt
+ * has ended; it is possible for this to be called multiple times while
+ * handling an ended connection attempt, and any future changes to this
+ * function must ensure it remains idempotent. */
+void
+rend_client_note_connection_attempt_ended(const rend_data_t *rend_data)
+{
+  unsigned int have_onion = 0;
+  rend_cache_entry_t *cache_entry = NULL;
+  const char *onion_address = rend_data_get_address(rend_data);
+  rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
+
+  if (onion_address[0] != '\0') {
+    /* Ignore return value; we find an entry, or we don't. */
+    (void) rend_cache_lookup_entry(onion_address, -1, &cache_entry);
+    have_onion = 1;
+  }
+
+  /* Clear the timed_out flag on all remaining intro points for this HS. */
+  if (cache_entry != NULL) {
+    SMARTLIST_FOREACH(cache_entry->parsed->intro_nodes,
+                      rend_intro_point_t *, ip,
+                      ip->timed_out = 0; );
+  }
+
+  /* Remove the HS's entries in last_hid_serv_requests. */
+  if (have_onion) {
+    unsigned int replica;
+    for (replica = 0; replica < ARRAY_LENGTH(rend_data_v2->descriptor_id);
+         replica++) {
+      const char *desc_id = rend_data_v2->descriptor_id[replica];
+      purge_v2_hidserv_req(desc_id);
+    }
+    log_info(LD_REND, "Connection attempt for %s has ended; "
+             "cleaning up temporary state.",
+             safe_str_client(onion_address));
+  } else {
+    /* We only have an ID for a fetch. Probably used by HSFETCH. */
+    purge_v2_hidserv_req(rend_data_v2->desc_id_fetch);
+  }
+}
+
+/** Return a newly allocated extend_info_t* for a randomly chosen introduction
+ * point for the named hidden service.  Return NULL if all introduction points
+ * have been tried and failed.
+ */
+extend_info_t *
+rend_client_get_random_intro(const rend_data_t *rend_query)
+{
+  int ret;
+  extend_info_t *result;
+  rend_cache_entry_t *entry;
+  const char *onion_address = rend_data_get_address(rend_query);
+
+  ret = rend_cache_lookup_entry(onion_address, -1, &entry);
+  if (ret < 0 || !rend_client_any_intro_points_usable(entry)) {
+    log_warn(LD_REND,
+             "Query '%s' didn't have valid rend desc in cache. Failing.",
+             safe_str_client(onion_address));
+    /* XXX: Should we refetch the descriptor here if the IPs are not usable
+     * anymore ?. */
+    return NULL;
+  }
+
+  /* See if we can get a node that complies with ExcludeNodes */
+  if ((result = rend_client_get_random_intro_impl(entry, 1, 1)))
+    return result;
+  /* If not, and StrictNodes is not set, see if we can return any old node
+   */
+  if (!get_options()->StrictNodes)
+    return rend_client_get_random_intro_impl(entry, 0, 1);
+  return NULL;
+}
+
+/** As rend_client_get_random_intro, except assume that StrictNodes is set
+ * iff <b>strict</b> is true. If <b>warnings</b> is false, don't complain
+ * to the user when we're out of nodes, even if StrictNodes is true.
+ */
+static extend_info_t *
+rend_client_get_random_intro_impl(const rend_cache_entry_t *entry,
+                                  const int strict,
+                                  const int warnings)
+{
+  int i;
+
+  rend_intro_point_t *intro;
+  const or_options_t *options = get_options();
+  smartlist_t *usable_nodes;
+  int n_excluded = 0;
+
+  /* We'll keep a separate list of the usable nodes.  If this becomes empty,
+   * no nodes are usable.  */
+  usable_nodes = smartlist_new();
+  smartlist_add_all(usable_nodes, entry->parsed->intro_nodes);
+
+  /* Remove the intro points that have timed out during this HS
+   * connection attempt from our list of usable nodes. */
+  SMARTLIST_FOREACH(usable_nodes, rend_intro_point_t *, ip,
+                    if (ip->timed_out) {
+                      SMARTLIST_DEL_CURRENT(usable_nodes, ip);
+                    });
+
+ again:
+  if (smartlist_len(usable_nodes) == 0) {
+    if (n_excluded && get_options()->StrictNodes && warnings) {
+      /* We only want to warn if StrictNodes is really set. Otherwise
+       * we're just about to retry anyways.
+       */
+      log_warn(LD_REND, "All introduction points for hidden service are "
+               "at excluded relays, and StrictNodes is set. Skipping.");
+    }
+    smartlist_free(usable_nodes);
+    return NULL;
+  }
+
+  i = crypto_rand_int(smartlist_len(usable_nodes));
+  intro = smartlist_get(usable_nodes, i);
+  if (BUG(!intro->extend_info)) {
+    /* This should never happen, but it isn't fatal, just try another */
+    smartlist_del(usable_nodes, i);
+    goto again;
+  }
+  /* All version 2 HS descriptors come with a TAP onion key.
+   * Clients used to try to get the TAP onion key from the consensus, but this
+   * meant that hidden services could discover which consensus clients have. */
+  if (!extend_info_supports_tap(intro->extend_info)) {
+    log_info(LD_REND, "The HS descriptor is missing a TAP onion key for the "
+             "intro-point relay '%s'; trying another.",
+             safe_str_client(extend_info_describe(intro->extend_info)));
+    smartlist_del(usable_nodes, i);
+    goto again;
+  }
+  /* Check if we should refuse to talk to this router. */
+  if (strict &&
+      routerset_contains_extendinfo(options->ExcludeNodes,
+                                    intro->extend_info)) {
+    n_excluded++;
+    smartlist_del(usable_nodes, i);
+    goto again;
+  }
+
+  smartlist_free(usable_nodes);
+  return extend_info_dup(intro->extend_info);
+}
+
+/** Return true iff any introduction points still listed in <b>entry</b> are
+ * usable. */
+int
+rend_client_any_intro_points_usable(const rend_cache_entry_t *entry)
+{
+  extend_info_t *extend_info =
+    rend_client_get_random_intro_impl(entry, get_options()->StrictNodes, 0);
+
+  int rv = (extend_info != NULL);
+
+  extend_info_free(extend_info);
+  return rv;
+}
+
+/** Client-side authorizations for hidden services; map of onion address to
+ * rend_service_authorization_t*. */
+static strmap_t *auth_hid_servs = NULL;
+
+/** Look up the client-side authorization for the hidden service with
+ * <b>onion_address</b>. Return NULL if no authorization is available for
+ * that address. */
+rend_service_authorization_t*
+rend_client_lookup_service_authorization(const char *onion_address)
+{
+  tor_assert(onion_address);
+  if (!auth_hid_servs) return NULL;
+  return strmap_get(auth_hid_servs, onion_address);
+}
+
+#define rend_service_authorization_free(val)                    \
+  FREE_AND_NULL(rend_service_authorization_t,                   \
+                rend_service_authorization_free_, (val))
+
+/** Helper: Free storage held by rend_service_authorization_t. */
+static void
+rend_service_authorization_free_(rend_service_authorization_t *auth)
+{
+  tor_free(auth);
+}
+
+/** Helper for strmap_free. */
+static void
+rend_service_authorization_free_void(void *service_auth)
+{
+  rend_service_authorization_free_(service_auth);
+}
+
+/** Release all the storage held in auth_hid_servs.
+ */
+void
+rend_service_authorization_free_all(void)
+{
+  if (!auth_hid_servs) {
+    return;
+  }
+  strmap_free(auth_hid_servs, rend_service_authorization_free_void);
+  auth_hid_servs = NULL;
+}
+
+/** Parse <b>config_line</b> as a client-side authorization for a hidden
+ * service and add it to the local map of hidden service authorizations.
+ * Return 0 for success and -1 for failure. */
+int
+rend_parse_service_authorization(const or_options_t *options,
+                                 int validate_only)
+{
+  config_line_t *line;
+  int res = -1;
+  strmap_t *parsed = strmap_new();
+  smartlist_t *sl = smartlist_new();
+  rend_service_authorization_t *auth = NULL;
+  char *err_msg = NULL;
+
+  for (line = options->HidServAuth; line; line = line->next) {
+    char *onion_address, *descriptor_cookie;
+    auth = NULL;
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c););
+    smartlist_clear(sl);
+    smartlist_split_string(sl, line->value, " ",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+    if (smartlist_len(sl) < 2) {
+      log_warn(LD_CONFIG, "Configuration line does not consist of "
+               "\"onion-address authorization-cookie [service-name]\": "
+               "'%s'", line->value);
+      goto err;
+    }
+    auth = tor_malloc_zero(sizeof(rend_service_authorization_t));
+    /* Parse onion address. */
+    onion_address = smartlist_get(sl, 0);
+    if (strlen(onion_address) != REND_SERVICE_ADDRESS_LEN ||
+        strcmpend(onion_address, ".onion")) {
+      log_warn(LD_CONFIG, "Onion address has wrong format: '%s'",
+               onion_address);
+      goto err;
+    }
+    strlcpy(auth->onion_address, onion_address, REND_SERVICE_ID_LEN_BASE32+1);
+    if (!rend_valid_v2_service_id(auth->onion_address)) {
+      log_warn(LD_CONFIG, "Onion address has wrong format: '%s'",
+               onion_address);
+      goto err;
+    }
+    /* Parse descriptor cookie. */
+    descriptor_cookie = smartlist_get(sl, 1);
+    if (rend_auth_decode_cookie(descriptor_cookie, auth->descriptor_cookie,
+                                &auth->auth_type, &err_msg) < 0) {
+      tor_assert(err_msg);
+      log_warn(LD_CONFIG, "%s", err_msg);
+      tor_free(err_msg);
+      goto err;
+    }
+    if (strmap_get(parsed, auth->onion_address)) {
+      log_warn(LD_CONFIG, "Duplicate authorization for the same hidden "
+                          "service.");
+      goto err;
+    }
+    strmap_set(parsed, auth->onion_address, auth);
+    auth = NULL;
+  }
+  res = 0;
+  goto done;
+ err:
+  res = -1;
+ done:
+  rend_service_authorization_free(auth);
+  SMARTLIST_FOREACH(sl, char *, c, tor_free(c););
+  smartlist_free(sl);
+  if (!validate_only && res == 0) {
+    rend_service_authorization_free_all();
+    auth_hid_servs = parsed;
+  } else {
+    strmap_free(parsed, rend_service_authorization_free_void);
+  }
+  return res;
+}
+
+/* Can Tor client code make direct (non-anonymous) connections to introduction
+ * or rendezvous points?
+ * Returns true if tor was compiled with NON_ANONYMOUS_MODE_ENABLED, and is
+ * configured in Tor2web mode. */
+int
+rend_client_allow_non_anonymous_connection(const or_options_t *options)
+{
+  /* Tor2web support needs to be compiled in to a tor binary. */
+#ifdef NON_ANONYMOUS_MODE_ENABLED
+  /* Tor2web */
+  return options->Tor2webMode ? 1 : 0;
+#else
+  (void)options;
+  return 0;
+#endif /* defined(NON_ANONYMOUS_MODE_ENABLED) */
+}
+
+/* At compile-time, was non-anonymous mode enabled via
+ * NON_ANONYMOUS_MODE_ENABLED ? */
+int
+rend_client_non_anonymous_mode_enabled(const or_options_t *options)
+{
+  (void)options;
+  /* Tor2web support needs to be compiled in to a tor binary. */
+#ifdef NON_ANONYMOUS_MODE_ENABLED
+  /* Tor2web */
+  return 1;
+#else
+  return 0;
+#endif /* defined(NON_ANONYMOUS_MODE_ENABLED) */
+}
diff --git a/src/feature/rend/rendclient.h b/src/feature/rend/rendclient.h
new file mode 100644
index 0000000000..77395d6cb8
--- /dev/null
+++ b/src/feature/rend/rendclient.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendclient.h
+ * \brief Header file for rendclient.c.
+ **/
+
+#ifndef TOR_RENDCLIENT_H
+#define TOR_RENDCLIENT_H
+
+#include "or/rendcache.h"
+
+void rend_client_purge_state(void);
+
+void rend_client_introcirc_has_opened(origin_circuit_t *circ);
+void rend_client_rendcirc_has_opened(origin_circuit_t *circ);
+int rend_client_introduction_acked(origin_circuit_t *circ,
+                                   const uint8_t *request,
+                                   size_t request_len);
+void rend_client_refetch_v2_renddesc(rend_data_t *rend_query);
+int rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs);
+void rend_client_cancel_descriptor_fetches(void);
+
+int rend_client_report_intro_point_failure(extend_info_t *failed_intro,
+                                           rend_data_t *rend_data,
+                                           unsigned int failure_type);
+
+int rend_client_receive_rendezvous(origin_circuit_t *circ,
+                                   const uint8_t *request,
+                                   size_t request_len);
+void rend_client_desc_trynow(const char *query);
+
+void rend_client_note_connection_attempt_ended(const rend_data_t *rend_data);
+
+extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query);
+int rend_client_any_intro_points_usable(const rend_cache_entry_t *entry);
+
+int rend_client_send_introduction(origin_circuit_t *introcirc,
+                                  origin_circuit_t *rendcirc);
+int rend_parse_service_authorization(const or_options_t *options,
+                                     int validate_only);
+rend_service_authorization_t *rend_client_lookup_service_authorization(
+                                                const char *onion_address);
+void rend_service_authorization_free_all(void);
+
+int rend_client_allow_non_anonymous_connection(const or_options_t *options);
+int rend_client_non_anonymous_mode_enabled(const or_options_t *options);
+
+#endif /* !defined(TOR_RENDCLIENT_H) */
+
diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c
new file mode 100644
index 0000000000..928dda0128
--- /dev/null
+++ b/src/feature/rend/rendcommon.c
@@ -0,0 +1,1056 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendcommon.c
+ * \brief Rendezvous implementation: shared code between
+ * introducers, services, clients, and rendezvous points.
+ **/
+
+#define RENDCOMMON_PRIVATE
+
+#include "or/or.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/hs_client.h"
+#include "or/hs_common.h"
+#include "or/hs_intropoint.h"
+#include "or/networkstatus.h"
+#include "or/rendclient.h"
+#include "or/rendcommon.h"
+#include "or/rendmid.h"
+#include "or/rendservice.h"
+#include "or/rephist.h"
+#include "or/replaycache.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+
+#include "or/cpath_build_state_st.h"
+#include "or/crypt_path_st.h"
+#include "or/extend_info_st.h"
+#include "or/networkstatus_st.h"
+#include "or/origin_circuit_st.h"
+#include "or/rend_encoded_v2_service_descriptor_st.h"
+#include "or/rend_intro_point_st.h"
+#include "or/rend_service_descriptor_st.h"
+#include "or/routerstatus_st.h"
+
+/** Return 0 if one and two are the same service ids, else -1 or 1 */
+int
+rend_cmp_service_ids(const char *one, const char *two)
+{
+  return strcasecmp(one,two);
+}
+
+/** Free the storage held by the service descriptor <b>desc</b>.
+ */
+void
+rend_service_descriptor_free_(rend_service_descriptor_t *desc)
+{
+  if (!desc)
+    return;
+  if (desc->pk)
+    crypto_pk_free(desc->pk);
+  if (desc->intro_nodes) {
+    SMARTLIST_FOREACH(desc->intro_nodes, rend_intro_point_t *, intro,
+      rend_intro_point_free(intro););
+    smartlist_free(desc->intro_nodes);
+  }
+  if (desc->successful_uploads) {
+    SMARTLIST_FOREACH(desc->successful_uploads, char *, c, tor_free(c););
+    smartlist_free(desc->successful_uploads);
+  }
+  tor_free(desc);
+}
+
+/** Length of the descriptor cookie that is used for versioned hidden
+ * service descriptors. */
+#define REND_DESC_COOKIE_LEN 16
+
+/** Length of the replica number that is used to determine the secret ID
+ * part of versioned hidden service descriptors. */
+#define REND_REPLICA_LEN 1
+
+/** Compute the descriptor ID for <b>service_id</b> of length
+ * <b>REND_SERVICE_ID_LEN</b> and <b>secret_id_part</b> of length
+ * <b>DIGEST_LEN</b>, and write it to <b>descriptor_id_out</b> of length
+ * <b>DIGEST_LEN</b>. */
+void
+rend_get_descriptor_id_bytes(char *descriptor_id_out,
+                             const char *service_id,
+                             const char *secret_id_part)
+{
+  crypto_digest_t *digest = crypto_digest_new();
+  crypto_digest_add_bytes(digest, service_id, REND_SERVICE_ID_LEN);
+  crypto_digest_add_bytes(digest, secret_id_part, DIGEST_LEN);
+  crypto_digest_get_digest(digest, descriptor_id_out, DIGEST_LEN);
+  crypto_digest_free(digest);
+}
+
+/** Compute the secret ID part for time_period,
+ * a <b>descriptor_cookie</b> of length
+ * <b>REND_DESC_COOKIE_LEN</b> which may also be <b>NULL</b> if no
+ * descriptor_cookie shall be used, and <b>replica</b>, and write it to
+ * <b>secret_id_part</b> of length DIGEST_LEN. */
+static void
+get_secret_id_part_bytes(char *secret_id_part, uint32_t time_period,
+                         const char *descriptor_cookie, uint8_t replica)
+{
+  crypto_digest_t *digest = crypto_digest_new();
+  time_period = htonl(time_period);
+  crypto_digest_add_bytes(digest, (char*)&time_period, sizeof(uint32_t));
+  if (descriptor_cookie) {
+    crypto_digest_add_bytes(digest, descriptor_cookie,
+                            REND_DESC_COOKIE_LEN);
+  }
+  crypto_digest_add_bytes(digest, (const char *)&replica, REND_REPLICA_LEN);
+  crypto_digest_get_digest(digest, secret_id_part, DIGEST_LEN);
+  crypto_digest_free(digest);
+}
+
+/** Return the time period for time <b>now</b> plus a potentially
+ * intended <b>deviation</b> of one or more periods, based on the first byte
+ * of <b>service_id</b>. */
+static uint32_t
+get_time_period(time_t now, uint8_t deviation, const char *service_id)
+{
+  /* The time period is the number of REND_TIME_PERIOD_V2_DESC_VALIDITY
+   * intervals that have passed since the epoch, offset slightly so that
+   * each service's time periods start and end at a fraction of that
+   * period based on their first byte. */
+  return (uint32_t)
+    (now + ((uint8_t) *service_id) * REND_TIME_PERIOD_V2_DESC_VALIDITY / 256)
+    / REND_TIME_PERIOD_V2_DESC_VALIDITY + deviation;
+}
+
+/** Compute the time in seconds that a descriptor that is generated
+ * <b>now</b> for <b>service_id</b> will be valid. */
+static uint32_t
+get_seconds_valid(time_t now, const char *service_id)
+{
+  uint32_t result = REND_TIME_PERIOD_V2_DESC_VALIDITY -
+    ((uint32_t)
+     (now + ((uint8_t) *service_id) * REND_TIME_PERIOD_V2_DESC_VALIDITY / 256)
+     % REND_TIME_PERIOD_V2_DESC_VALIDITY);
+  return result;
+}
+
+/** Compute the binary <b>desc_id_out</b> (DIGEST_LEN bytes long) for a given
+ * base32-encoded <b>service_id</b> and optional unencoded
+ * <b>descriptor_cookie</b> of length REND_DESC_COOKIE_LEN,
+ * at time <b>now</b> for replica number
+ * <b>replica</b>. <b>desc_id</b> needs to have <b>DIGEST_LEN</b> bytes
+ * free. Return 0 for success, -1 otherwise. */
+int
+rend_compute_v2_desc_id(char *desc_id_out, const char *service_id,
+                        const char *descriptor_cookie, time_t now,
+                        uint8_t replica)
+{
+  char service_id_binary[REND_SERVICE_ID_LEN];
+  char secret_id_part[DIGEST_LEN];
+  uint32_t time_period;
+  if (!service_id ||
+      strlen(service_id) != REND_SERVICE_ID_LEN_BASE32) {
+    log_warn(LD_REND, "Could not compute v2 descriptor ID: "
+                      "Illegal service ID: %s",
+             safe_str(service_id));
+    return -1;
+  }
+  if (replica >= REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS) {
+    log_warn(LD_REND, "Could not compute v2 descriptor ID: "
+                      "Replica number out of range: %d", replica);
+    return -1;
+  }
+  /* Convert service ID to binary. */
+  if (base32_decode(service_id_binary, REND_SERVICE_ID_LEN,
+                    service_id, REND_SERVICE_ID_LEN_BASE32) < 0) {
+    log_warn(LD_REND, "Could not compute v2 descriptor ID: "
+                      "Illegal characters in service ID: %s",
+             safe_str_client(service_id));
+    return -1;
+  }
+  /* Calculate current time-period. */
+  time_period = get_time_period(now, 0, service_id_binary);
+  /* Calculate secret-id-part = h(time-period | desc-cookie | replica). */
+  get_secret_id_part_bytes(secret_id_part, time_period, descriptor_cookie,
+                           replica);
+  /* Calculate descriptor ID: H(permanent-id | secret-id-part) */
+  rend_get_descriptor_id_bytes(desc_id_out, service_id_binary, secret_id_part);
+  return 0;
+}
+
+/** Encode the introduction points in <b>desc</b> and write the result to a
+ * newly allocated string pointed to by <b>encoded</b>. Return 0 for
+ * success, -1 otherwise. */
+static int
+rend_encode_v2_intro_points(char **encoded, rend_service_descriptor_t *desc)
+{
+  size_t unenc_len;
+  char *unenc = NULL;
+  size_t unenc_written = 0;
+  int i;
+  int r = -1;
+  /* Assemble unencrypted list of introduction points. */
+  unenc_len = smartlist_len(desc->intro_nodes) * 1000; /* too long, but ok. */
+  unenc = tor_malloc_zero(unenc_len);
+  for (i = 0; i < smartlist_len(desc->intro_nodes); i++) {
+    char id_base32[REND_INTRO_POINT_ID_LEN_BASE32 + 1];
+    char *onion_key = NULL;
+    size_t onion_key_len;
+    crypto_pk_t *intro_key;
+    char *service_key = NULL;
+    char *address = NULL;
+    size_t service_key_len;
+    int res;
+    rend_intro_point_t *intro = smartlist_get(desc->intro_nodes, i);
+    /* Obtain extend info with introduction point details. */
+    extend_info_t *info = intro->extend_info;
+    /* Encode introduction point ID. */
+    base32_encode(id_base32, sizeof(id_base32),
+                  info->identity_digest, DIGEST_LEN);
+    /* Encode onion key. */
+    if (crypto_pk_write_public_key_to_string(info->onion_key, &onion_key,
+                                             &onion_key_len) < 0) {
+      log_warn(LD_REND, "Could not write onion key.");
+      goto done;
+    }
+    /* Encode intro key. */
+    intro_key = intro->intro_key;
+    if (!intro_key ||
+      crypto_pk_write_public_key_to_string(intro_key, &service_key,
+                                           &service_key_len) < 0) {
+      log_warn(LD_REND, "Could not write intro key.");
+      tor_free(onion_key);
+      goto done;
+    }
+    /* Assemble everything for this introduction point. */
+    address = tor_addr_to_str_dup(&info->addr);
+    res = tor_snprintf(unenc + unenc_written, unenc_len - unenc_written,
+                         "introduction-point %s\n"
+                         "ip-address %s\n"
+                         "onion-port %d\n"
+                         "onion-key\n%s"
+                         "service-key\n%s",
+                       id_base32,
+                       address,
+                       info->port,
+                       onion_key,
+                       service_key);
+    tor_free(address);
+    tor_free(onion_key);
+    tor_free(service_key);
+    if (res < 0) {
+      log_warn(LD_REND, "Not enough space for writing introduction point "
+                        "string.");
+      goto done;
+    }
+    /* Update total number of written bytes for unencrypted intro points. */
+    unenc_written += res;
+  }
+  /* Finalize unencrypted introduction points. */
+  if (unenc_len < unenc_written + 2) {
+    log_warn(LD_REND, "Not enough space for finalizing introduction point "
+                      "string.");
+    goto done;
+  }
+  unenc[unenc_written++] = '\n';
+  unenc[unenc_written++] = 0;
+  *encoded = unenc;
+  r = 0;
+ done:
+  if (r<0)
+    tor_free(unenc);
+  return r;
+}
+
+/** Encrypt the encoded introduction points in <b>encoded</b> using
+ * authorization type  'basic' with <b>client_cookies</b> and write the
+ * result to a newly allocated string pointed to by <b>encrypted_out</b> of
+ * length <b>encrypted_len_out</b>. Return 0 for success, -1 otherwise. */
+static int
+rend_encrypt_v2_intro_points_basic(char **encrypted_out,
+                                   size_t *encrypted_len_out,
+                                   const char *encoded,
+                                   smartlist_t *client_cookies)
+{
+  int r = -1, i, pos, enclen, client_blocks;
+  size_t len, client_entries_len;
+  char *enc = NULL, iv[CIPHER_IV_LEN], *client_part = NULL,
+       session_key[CIPHER_KEY_LEN];
+  smartlist_t *encrypted_session_keys = NULL;
+  crypto_digest_t *digest;
+  crypto_cipher_t *cipher;
+  tor_assert(encoded);
+  tor_assert(client_cookies && smartlist_len(client_cookies) > 0);
+
+  /* Generate session key. */
+  crypto_rand(session_key, CIPHER_KEY_LEN);
+
+  /* Determine length of encrypted introduction points including session
+   * keys. */
+  client_blocks = 1 + ((smartlist_len(client_cookies) - 1) /
+                       REND_BASIC_AUTH_CLIENT_MULTIPLE);
+  client_entries_len = client_blocks * REND_BASIC_AUTH_CLIENT_MULTIPLE *
+                       REND_BASIC_AUTH_CLIENT_ENTRY_LEN;
+  len = 2 + client_entries_len + CIPHER_IV_LEN + strlen(encoded);
+  if (client_blocks >= 256) {
+    log_warn(LD_REND, "Too many clients in introduction point string.");
+    goto done;
+  }
+  enc = tor_malloc_zero(len);
+  enc[0] = 0x01; /* type of authorization. */
+  enc[1] = (uint8_t)client_blocks;
+
+  /* Encrypt with random session key. */
+  enclen = crypto_cipher_encrypt_with_iv(session_key,
+      enc + 2 + client_entries_len,
+      CIPHER_IV_LEN + strlen(encoded), encoded, strlen(encoded));
+
+  if (enclen < 0) {
+    log_warn(LD_REND, "Could not encrypt introduction point string.");
+    goto done;
+  }
+  memcpy(iv, enc + 2 + client_entries_len, CIPHER_IV_LEN);
+
+  /* Encrypt session key for cookies, determine client IDs, and put both
+   * in a smartlist. */
+  encrypted_session_keys = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(client_cookies, const char *, cookie) {
+    client_part = tor_malloc_zero(REND_BASIC_AUTH_CLIENT_ENTRY_LEN);
+    /* Encrypt session key. */
+    cipher = crypto_cipher_new(cookie);
+    if (crypto_cipher_encrypt(cipher, client_part +
+                                  REND_BASIC_AUTH_CLIENT_ID_LEN,
+                              session_key, CIPHER_KEY_LEN) < 0) {
+      log_warn(LD_REND, "Could not encrypt session key for client.");
+      crypto_cipher_free(cipher);
+      tor_free(client_part);
+      goto done;
+    }
+    crypto_cipher_free(cipher);
+
+    /* Determine client ID. */
+    digest = crypto_digest_new();
+    crypto_digest_add_bytes(digest, cookie, REND_DESC_COOKIE_LEN);
+    crypto_digest_add_bytes(digest, iv, CIPHER_IV_LEN);
+    crypto_digest_get_digest(digest, client_part,
+                             REND_BASIC_AUTH_CLIENT_ID_LEN);
+    crypto_digest_free(digest);
+
+    /* Put both together. */
+    smartlist_add(encrypted_session_keys, client_part);
+  } SMARTLIST_FOREACH_END(cookie);
+
+  /* Add some fake client IDs and encrypted session keys. */
+  for (i = (smartlist_len(client_cookies) - 1) %
+           REND_BASIC_AUTH_CLIENT_MULTIPLE;
+       i < REND_BASIC_AUTH_CLIENT_MULTIPLE - 1; i++) {
+    client_part = tor_malloc_zero(REND_BASIC_AUTH_CLIENT_ENTRY_LEN);
+    crypto_rand(client_part, REND_BASIC_AUTH_CLIENT_ENTRY_LEN);
+    smartlist_add(encrypted_session_keys, client_part);
+  }
+  /* Sort smartlist and put elements in result in order. */
+  smartlist_sort_digests(encrypted_session_keys);
+  pos = 2;
+  SMARTLIST_FOREACH(encrypted_session_keys, const char *, entry, {
+    memcpy(enc + pos, entry, REND_BASIC_AUTH_CLIENT_ENTRY_LEN);
+    pos += REND_BASIC_AUTH_CLIENT_ENTRY_LEN;
+  });
+  *encrypted_out = enc;
+  *encrypted_len_out = len;
+  enc = NULL; /* prevent free. */
+  r = 0;
+ done:
+  tor_free(enc);
+  if (encrypted_session_keys) {
+    SMARTLIST_FOREACH(encrypted_session_keys, char *, d, tor_free(d););
+    smartlist_free(encrypted_session_keys);
+  }
+  return r;
+}
+
+/** Encrypt the encoded introduction points in <b>encoded</b> using
+ * authorization type 'stealth' with <b>descriptor_cookie</b> of length
+ * REND_DESC_COOKIE_LEN and write the result to a newly allocated string
+ * pointed to by <b>encrypted_out</b> of length <b>encrypted_len_out</b>.
+ * Return 0 for success, -1 otherwise. */
+static int
+rend_encrypt_v2_intro_points_stealth(char **encrypted_out,
+                                     size_t *encrypted_len_out,
+                                     const char *encoded,
+                                     const char *descriptor_cookie)
+{
+  int r = -1, enclen;
+  char *enc;
+  tor_assert(encoded);
+  tor_assert(descriptor_cookie);
+
+  enc = tor_malloc_zero(1 + CIPHER_IV_LEN + strlen(encoded));
+  enc[0] = 0x02; /* Auth type */
+  enclen = crypto_cipher_encrypt_with_iv(descriptor_cookie,
+                                         enc + 1,
+                                         CIPHER_IV_LEN+strlen(encoded),
+                                         encoded, strlen(encoded));
+  if (enclen < 0) {
+    log_warn(LD_REND, "Could not encrypt introduction point string.");
+    goto done;
+  }
+  *encrypted_out = enc;
+  *encrypted_len_out = enclen;
+  enc = NULL; /* prevent free */
+  r = 0;
+ done:
+  tor_free(enc);
+  return r;
+}
+
+/** Attempt to parse the given <b>desc_str</b> and return true if this
+ * succeeds, false otherwise. */
+STATIC int
+rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc)
+{
+  rend_service_descriptor_t *test_parsed = NULL;
+  char test_desc_id[DIGEST_LEN];
+  char *test_intro_content = NULL;
+  size_t test_intro_size;
+  size_t test_encoded_size;
+  const char *test_next;
+  int res = rend_parse_v2_service_descriptor(&test_parsed, test_desc_id,
+                                         &test_intro_content,
+                                         &test_intro_size,
+                                         &test_encoded_size,
+                                         &test_next, desc->desc_str, 1);
+  rend_service_descriptor_free(test_parsed);
+  tor_free(test_intro_content);
+  return (res >= 0);
+}
+
+/** Free the storage held by an encoded v2 service descriptor. */
+void
+rend_encoded_v2_service_descriptor_free_(
+  rend_encoded_v2_service_descriptor_t *desc)
+{
+  if (!desc)
+    return;
+  tor_free(desc->desc_str);
+  tor_free(desc);
+}
+
+/** Free the storage held by an introduction point info. */
+void
+rend_intro_point_free_(rend_intro_point_t *intro)
+{
+  if (!intro)
+    return;
+
+  extend_info_free(intro->extend_info);
+  crypto_pk_free(intro->intro_key);
+
+  if (intro->accepted_intro_rsa_parts != NULL) {
+    replaycache_free(intro->accepted_intro_rsa_parts);
+  }
+
+  tor_free(intro);
+}
+
+/** Encode a set of rend_encoded_v2_service_descriptor_t's for <b>desc</b>
+ * at time <b>now</b> using <b>service_key</b>, depending on
+ * <b>auth_type</b> a <b>descriptor_cookie</b> and a list of
+ * <b>client_cookies</b> (which are both <b>NULL</b> if no client
+ * authorization is performed), and <b>period</b> (e.g. 0 for the current
+ * period, 1 for the next period, etc.) and add them to the existing list
+ * <b>descs_out</b>; return the number of seconds that the descriptors will
+ * be found by clients, or -1 if the encoding was not successful. */
+int
+rend_encode_v2_descriptors(smartlist_t *descs_out,
+                           rend_service_descriptor_t *desc, time_t now,
+                           uint8_t period, rend_auth_type_t auth_type,
+                           crypto_pk_t *client_key,
+                           smartlist_t *client_cookies)
+{
+  char service_id[DIGEST_LEN];
+  char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1];
+  uint32_t time_period;
+  char *ipos_base64 = NULL, *ipos = NULL, *ipos_encrypted = NULL,
+       *descriptor_cookie = NULL;
+  size_t ipos_len = 0, ipos_encrypted_len = 0;
+  int k;
+  uint32_t seconds_valid;
+  crypto_pk_t *service_key;
+  if (!desc) {
+    log_warn(LD_BUG, "Could not encode v2 descriptor: No desc given.");
+    return -1;
+  }
+  service_key = (auth_type == REND_STEALTH_AUTH) ? client_key : desc->pk;
+  tor_assert(service_key);
+  if (auth_type == REND_STEALTH_AUTH) {
+    descriptor_cookie = smartlist_get(client_cookies, 0);
+    tor_assert(descriptor_cookie);
+  }
+  /* Obtain service_id from public key. */
+  if (crypto_pk_get_digest(service_key, service_id) < 0) {
+    log_warn(LD_BUG, "Couldn't compute service key digest.");
+    return -1;
+  }
+  /* Calculate current time-period. */
+  time_period = get_time_period(now, period, service_id);
+  /* Determine how many seconds the descriptor will be valid. */
+  seconds_valid = period * REND_TIME_PERIOD_V2_DESC_VALIDITY +
+                  get_seconds_valid(now, service_id);
+  /* Assemble, possibly encrypt, and encode introduction points. */
+  if (smartlist_len(desc->intro_nodes) > 0) {
+    if (rend_encode_v2_intro_points(&ipos, desc) < 0) {
+      log_warn(LD_REND, "Encoding of introduction points did not succeed.");
+      return -1;
+    }
+    switch (auth_type) {
+      case REND_NO_AUTH:
+        ipos_len = strlen(ipos);
+        break;
+      case REND_BASIC_AUTH:
+        if (rend_encrypt_v2_intro_points_basic(&ipos_encrypted,
+                                               &ipos_encrypted_len, ipos,
+                                               client_cookies) < 0) {
+          log_warn(LD_REND, "Encrypting of introduction points did not "
+                            "succeed.");
+          tor_free(ipos);
+          return -1;
+        }
+        tor_free(ipos);
+        ipos = ipos_encrypted;
+        ipos_len = ipos_encrypted_len;
+        break;
+      case REND_STEALTH_AUTH:
+        if (rend_encrypt_v2_intro_points_stealth(&ipos_encrypted,
+                                                 &ipos_encrypted_len, ipos,
+                                                 descriptor_cookie) < 0) {
+          log_warn(LD_REND, "Encrypting of introduction points did not "
+                            "succeed.");
+          tor_free(ipos);
+          return -1;
+        }
+        tor_free(ipos);
+        ipos = ipos_encrypted;
+        ipos_len = ipos_encrypted_len;
+        break;
+      default:
+        log_warn(LD_REND|LD_BUG, "Unrecognized authorization type %d",
+                 (int)auth_type);
+        tor_free(ipos);
+        return -1;
+    }
+    /* Base64-encode introduction points. */
+    ipos_base64 = tor_calloc(ipos_len, 2);
+    if (base64_encode(ipos_base64, ipos_len * 2, ipos, ipos_len,
+                      BASE64_ENCODE_MULTILINE)<0) {
+      log_warn(LD_REND, "Could not encode introduction point string to "
+               "base64. length=%d", (int)ipos_len);
+      tor_free(ipos_base64);
+      tor_free(ipos);
+      return -1;
+    }
+    tor_free(ipos);
+  }
+  /* Encode REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS descriptors. */
+  for (k = 0; k < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; k++) {
+    char secret_id_part[DIGEST_LEN];
+    char secret_id_part_base32[REND_SECRET_ID_PART_LEN_BASE32 + 1];
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    char *permanent_key = NULL;
+    size_t permanent_key_len;
+    char published[ISO_TIME_LEN+1];
+    int i;
+    char protocol_versions_string[16]; /* max len: "0,1,2,3,4,5,6,7\0" */
+    size_t protocol_versions_written;
+    size_t desc_len;
+    char *desc_str = NULL;
+    int result = 0;
+    size_t written = 0;
+    char desc_digest[DIGEST_LEN];
+    rend_encoded_v2_service_descriptor_t *enc =
+      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
+    /* Calculate secret-id-part = h(time-period | cookie | replica). */
+    get_secret_id_part_bytes(secret_id_part, time_period, descriptor_cookie,
+                             k);
+    base32_encode(secret_id_part_base32, sizeof(secret_id_part_base32),
+                  secret_id_part, DIGEST_LEN);
+    /* Calculate descriptor ID. */
+    rend_get_descriptor_id_bytes(enc->desc_id, service_id, secret_id_part);
+    base32_encode(desc_id_base32, sizeof(desc_id_base32),
+                  enc->desc_id, DIGEST_LEN);
+    /* PEM-encode the public key */
+    if (crypto_pk_write_public_key_to_string(service_key, &permanent_key,
+                                             &permanent_key_len) < 0) {
+      log_warn(LD_BUG, "Could not write public key to string.");
+      rend_encoded_v2_service_descriptor_free(enc);
+      goto err;
+    }
+    /* Encode timestamp. */
+    format_iso_time(published, desc->timestamp);
+    /* Write protocol-versions bitmask to comma-separated value string. */
+    protocol_versions_written = 0;
+    for (i = 0; i < 8; i++) {
+      if (desc->protocols & 1 << i) {
+        tor_snprintf(protocol_versions_string + protocol_versions_written,
+                     16 - protocol_versions_written, "%d,", i);
+        protocol_versions_written += 2;
+      }
+    }
+    if (protocol_versions_written)
+      protocol_versions_string[protocol_versions_written - 1] = '\0';
+    else
+      protocol_versions_string[0]= '\0';
+    /* Assemble complete descriptor. */
+    desc_len = 2000 + smartlist_len(desc->intro_nodes) * 1000; /* far too long,
+                                                                  but okay.*/
+    enc->desc_str = desc_str = tor_malloc_zero(desc_len);
+    result = tor_snprintf(desc_str, desc_len,
+             "rendezvous-service-descriptor %s\n"
+             "version 2\n"
+             "permanent-key\n%s"
+             "secret-id-part %s\n"
+             "publication-time %s\n"
+             "protocol-versions %s\n",
+        desc_id_base32,
+        permanent_key,
+        secret_id_part_base32,
+        published,
+        protocol_versions_string);
+    tor_free(permanent_key);
+    if (result < 0) {
+      log_warn(LD_BUG, "Descriptor ran out of room.");
+      rend_encoded_v2_service_descriptor_free(enc);
+      goto err;
+    }
+    written = result;
+    /* Add introduction points. */
+    if (ipos_base64) {
+      result = tor_snprintf(desc_str + written, desc_len - written,
+                            "introduction-points\n"
+                            "-----BEGIN MESSAGE-----\n%s"
+                            "-----END MESSAGE-----\n",
+                            ipos_base64);
+      if (result < 0) {
+        log_warn(LD_BUG, "could not write introduction points.");
+        rend_encoded_v2_service_descriptor_free(enc);
+        goto err;
+      }
+      written += result;
+    }
+    /* Add signature. */
+    strlcpy(desc_str + written, "signature\n", desc_len - written);
+    written += strlen(desc_str + written);
+    if (crypto_digest(desc_digest, desc_str, written) < 0) {
+      log_warn(LD_BUG, "could not create digest.");
+      rend_encoded_v2_service_descriptor_free(enc);
+      goto err;
+    }
+    if (router_append_dirobj_signature(desc_str + written,
+                                       desc_len - written,
+                                       desc_digest, DIGEST_LEN,
+                                       service_key) < 0) {
+      log_warn(LD_BUG, "Couldn't sign desc.");
+      rend_encoded_v2_service_descriptor_free(enc);
+      goto err;
+    }
+    written += strlen(desc_str+written);
+    if (written+2 > desc_len) {
+        log_warn(LD_BUG, "Could not finish desc.");
+        rend_encoded_v2_service_descriptor_free(enc);
+        goto err;
+    }
+    desc_str[written++] = 0;
+    /* Check if we can parse our own descriptor. */
+    if (!rend_desc_v2_is_parsable(enc)) {
+      log_warn(LD_BUG, "Could not parse my own descriptor: %s", desc_str);
+      rend_encoded_v2_service_descriptor_free(enc);
+      goto err;
+    }
+    smartlist_add(descs_out, enc);
+    /* Add the uploaded descriptor to the local service's descriptor cache */
+    rend_cache_store_v2_desc_as_service(enc->desc_str);
+    base32_encode(service_id_base32, sizeof(service_id_base32),
+          service_id, REND_SERVICE_ID_LEN);
+    control_event_hs_descriptor_created(service_id_base32, desc_id_base32, k);
+  }
+
+  log_info(LD_REND, "Successfully encoded a v2 descriptor and "
+                    "confirmed that it is parsable.");
+  goto done;
+
+ err:
+  SMARTLIST_FOREACH(descs_out, rend_encoded_v2_service_descriptor_t *, d,
+                    rend_encoded_v2_service_descriptor_free(d););
+  smartlist_clear(descs_out);
+  seconds_valid = -1;
+
+ done:
+  tor_free(ipos_base64);
+  return seconds_valid;
+}
+
+/** Sets <b>out</b> to the first 10 bytes of the digest of <b>pk</b>,
+ * base32 encoded.  NUL-terminates out.  (We use this string to
+ * identify services in directory requests and .onion URLs.)
+ */
+int
+rend_get_service_id(crypto_pk_t *pk, char *out)
+{
+  char buf[DIGEST_LEN];
+  tor_assert(pk);
+  if (crypto_pk_get_digest(pk, buf) < 0)
+    return -1;
+  base32_encode(out, REND_SERVICE_ID_LEN_BASE32+1, buf, REND_SERVICE_ID_LEN);
+  return 0;
+}
+
+/** Return true iff <b>query</b> is a syntactically valid service ID (as
+ * generated by rend_get_service_id).  */
+int
+rend_valid_v2_service_id(const char *query)
+{
+  if (strlen(query) != REND_SERVICE_ID_LEN_BASE32)
+    return 0;
+
+  if (strspn(query, BASE32_CHARS) != REND_SERVICE_ID_LEN_BASE32)
+    return 0;
+
+  return 1;
+}
+
+/** Return true iff <b>query</b> is a syntactically valid descriptor ID.
+ * (as generated by rend_get_descriptor_id_bytes). */
+int
+rend_valid_descriptor_id(const char *query)
+{
+  if (strlen(query) != REND_DESC_ID_V2_LEN_BASE32) {
+    goto invalid;
+  }
+  if (strspn(query, BASE32_CHARS) != REND_DESC_ID_V2_LEN_BASE32) {
+    goto invalid;
+  }
+
+  return 1;
+
+ invalid:
+  return 0;
+}
+
+/** Return true iff <b>client_name</b> is a syntactically valid name
+ * for rendezvous client authentication. */
+int
+rend_valid_client_name(const char *client_name)
+{
+  size_t len = strlen(client_name);
+  if (len < 1 || len > REND_CLIENTNAME_MAX_LEN) {
+    return 0;
+  }
+  if (strspn(client_name, REND_LEGAL_CLIENTNAME_CHARACTERS) != len) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/** Called when we get a rendezvous-related relay cell on circuit
+ * <b>circ</b>.  Dispatch on rendezvous relay command. */
+void
+rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
+                        int command, size_t length,
+                        const uint8_t *payload)
+{
+  or_circuit_t *or_circ = NULL;
+  origin_circuit_t *origin_circ = NULL;
+  int r = -2;
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    origin_circ = TO_ORIGIN_CIRCUIT(circ);
+    if (!layer_hint || layer_hint != origin_circ->cpath->prev) {
+      log_fn(LOG_PROTOCOL_WARN, LD_APP,
+             "Relay cell (rend purpose %d) from wrong hop on origin circ",
+             command);
+      origin_circ = NULL;
+    }
+  } else {
+    or_circ = TO_OR_CIRCUIT(circ);
+  }
+
+  switch (command) {
+    case RELAY_COMMAND_ESTABLISH_INTRO:
+      if (or_circ)
+        r = hs_intro_received_establish_intro(or_circ,payload,length);
+      break;
+    case RELAY_COMMAND_ESTABLISH_RENDEZVOUS:
+      if (or_circ)
+        r = rend_mid_establish_rendezvous(or_circ,payload,length);
+      break;
+    case RELAY_COMMAND_INTRODUCE1:
+      if (or_circ)
+        r = hs_intro_received_introduce1(or_circ,payload,length);
+      break;
+    case RELAY_COMMAND_INTRODUCE2:
+      if (origin_circ)
+        r = hs_service_receive_introduce2(origin_circ,payload,length);
+      break;
+    case RELAY_COMMAND_INTRODUCE_ACK:
+      if (origin_circ)
+        r = hs_client_receive_introduce_ack(origin_circ,payload,length);
+      break;
+    case RELAY_COMMAND_RENDEZVOUS1:
+      if (or_circ)
+        r = rend_mid_rendezvous(or_circ,payload,length);
+      break;
+    case RELAY_COMMAND_RENDEZVOUS2:
+      if (origin_circ)
+        r = hs_client_receive_rendezvous2(origin_circ,payload,length);
+      break;
+    case RELAY_COMMAND_INTRO_ESTABLISHED:
+      if (origin_circ)
+        r = hs_service_receive_intro_established(origin_circ,payload,length);
+      break;
+    case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
+      if (origin_circ)
+        r = hs_client_receive_rendezvous_acked(origin_circ,payload,length);
+      break;
+    default:
+      tor_fragile_assert();
+  }
+
+  if (r == 0 && origin_circ) {
+    /* This was a valid cell. Count it as delivered + overhead. */
+    circuit_read_valid_data(origin_circ, length);
+  }
+
+  if (r == -2)
+    log_info(LD_PROTOCOL, "Dropping cell (type %d) for wrong circuit type.",
+             command);
+}
+
+/** Determine the routers that are responsible for <b>id</b> (binary) and
+ * add pointers to those routers' routerstatus_t to <b>responsible_dirs</b>.
+ * Return -1 if we're returning an empty smartlist, else return 0.
+ */
+int
+hid_serv_get_responsible_directories(smartlist_t *responsible_dirs,
+                                     const char *id)
+{
+  int start, found, n_added = 0, i;
+  networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (!c || !smartlist_len(c->routerstatus_list)) {
+    log_warn(LD_REND, "We don't have a consensus, so we can't perform v2 "
+             "rendezvous operations.");
+    return -1;
+  }
+  tor_assert(id);
+  start = networkstatus_vote_find_entry_idx(c, id, &found);
+  if (start == smartlist_len(c->routerstatus_list)) start = 0;
+  i = start;
+  do {
+    routerstatus_t *r = smartlist_get(c->routerstatus_list, i);
+    if (r->is_hs_dir) {
+      smartlist_add(responsible_dirs, r);
+      if (++n_added == REND_NUMBER_OF_CONSECUTIVE_REPLICAS)
+        return 0;
+    }
+    if (++i == smartlist_len(c->routerstatus_list))
+      i = 0;
+  } while (i != start);
+
+  /* Even though we don't have the desired number of hidden service
+   * directories, be happy if we got any. */
+  return smartlist_len(responsible_dirs) ? 0 : -1;
+}
+
+/* Length of the 'extended' auth cookie used to encode auth type before
+ * base64 encoding. */
+#define REND_DESC_COOKIE_LEN_EXT (REND_DESC_COOKIE_LEN + 1)
+/* Length of the zero-padded auth cookie when base64 encoded. These two
+ * padding bytes always (A=) are stripped off of the returned cookie. */
+#define REND_DESC_COOKIE_LEN_EXT_BASE64 (REND_DESC_COOKIE_LEN_BASE64 + 2)
+
+/** Encode a client authorization descriptor cookie.
+ * The result of this function is suitable for use in the HidServAuth
+ * option.  The trailing padding characters are removed, and the
+ * auth type is encoded into the cookie.
+ *
+ * Returns a new base64-encoded cookie. This function cannot fail.
+ * The caller is responsible for freeing the returned value.
+ */
+char *
+rend_auth_encode_cookie(const uint8_t *cookie_in, rend_auth_type_t auth_type)
+{
+  uint8_t extended_cookie[REND_DESC_COOKIE_LEN_EXT];
+  char *cookie_out = tor_malloc_zero(REND_DESC_COOKIE_LEN_EXT_BASE64 + 1);
+  int re;
+
+  tor_assert(cookie_in);
+
+  memcpy(extended_cookie, cookie_in, REND_DESC_COOKIE_LEN);
+  extended_cookie[REND_DESC_COOKIE_LEN] = ((int)auth_type - 1) << 4;
+  re = base64_encode(cookie_out, REND_DESC_COOKIE_LEN_EXT_BASE64 + 1,
+                     (const char *) extended_cookie, REND_DESC_COOKIE_LEN_EXT,
+                     0);
+  tor_assert(re == REND_DESC_COOKIE_LEN_EXT_BASE64);
+
+  /* Remove the trailing 'A='.  Auth type is encoded in the high bits
+   * of the last byte, so the last base64 character will always be zero
+   * (A).  This is subtly different behavior from base64_encode_nopad. */
+  cookie_out[REND_DESC_COOKIE_LEN_BASE64] = '\0';
+  memwipe(extended_cookie, 0, sizeof(extended_cookie));
+  return cookie_out;
+}
+
+/** Decode a base64-encoded client authorization descriptor cookie.
+ * The descriptor_cookie can be truncated to REND_DESC_COOKIE_LEN_BASE64
+ * characters (as given to clients), or may include the two padding
+ * characters (as stored by the service).
+ *
+ * The result is stored in REND_DESC_COOKIE_LEN bytes of cookie_out.
+ * The rend_auth_type_t decoded from the cookie is stored in the
+ * optional auth_type_out parameter.
+ *
+ * Return 0 on success, or -1 on error.  The caller is responsible for
+ * freeing the returned err_msg.
+ */
+int
+rend_auth_decode_cookie(const char *cookie_in, uint8_t *cookie_out,
+                        rend_auth_type_t *auth_type_out, char **err_msg_out)
+{
+  uint8_t descriptor_cookie_decoded[REND_DESC_COOKIE_LEN_EXT + 1] = { 0 };
+  char descriptor_cookie_base64ext[REND_DESC_COOKIE_LEN_EXT_BASE64 + 1];
+  const char *descriptor_cookie = cookie_in;
+  char *err_msg = NULL;
+  int auth_type_val = 0;
+  int res = -1;
+  int decoded_len;
+
+  size_t len = strlen(descriptor_cookie);
+  if (len == REND_DESC_COOKIE_LEN_BASE64) {
+    /* Add a trailing zero byte to make base64-decoding happy. */
+    tor_snprintf(descriptor_cookie_base64ext,
+                 sizeof(descriptor_cookie_base64ext),
+                 "%sA=", descriptor_cookie);
+    descriptor_cookie = descriptor_cookie_base64ext;
+  } else if (len != REND_DESC_COOKIE_LEN_EXT_BASE64) {
+    tor_asprintf(&err_msg, "Authorization cookie has wrong length: %s",
+                 escaped(cookie_in));
+    goto err;
+  }
+
+  decoded_len = base64_decode((char *) descriptor_cookie_decoded,
+                              sizeof(descriptor_cookie_decoded),
+                              descriptor_cookie,
+                              REND_DESC_COOKIE_LEN_EXT_BASE64);
+  if (decoded_len != REND_DESC_COOKIE_LEN &&
+      decoded_len != REND_DESC_COOKIE_LEN_EXT) {
+    tor_asprintf(&err_msg, "Authorization cookie has invalid characters: %s",
+                 escaped(cookie_in));
+    goto err;
+  }
+
+  if (auth_type_out) {
+    auth_type_val = (descriptor_cookie_decoded[REND_DESC_COOKIE_LEN] >> 4) + 1;
+    if (auth_type_val < 1 || auth_type_val > 2) {
+      tor_asprintf(&err_msg, "Authorization cookie type is unknown: %s",
+                   escaped(cookie_in));
+      goto err;
+    }
+    *auth_type_out = auth_type_val == 1 ? REND_BASIC_AUTH : REND_STEALTH_AUTH;
+  }
+
+  memcpy(cookie_out, descriptor_cookie_decoded, REND_DESC_COOKIE_LEN);
+  res = 0;
+ err:
+  if (err_msg_out) {
+    *err_msg_out = err_msg;
+  } else {
+    tor_free(err_msg);
+  }
+  memwipe(descriptor_cookie_decoded, 0, sizeof(descriptor_cookie_decoded));
+  memwipe(descriptor_cookie_base64ext, 0, sizeof(descriptor_cookie_base64ext));
+  return res;
+}
+
+/* Is this a rend client or server that allows direct (non-anonymous)
+ * connections?
+ * Clients must be specifically compiled and configured in this mode.
+ * Onion services can be configured to start in this mode.
+ * Prefer rend_client_allow_non_anonymous_connection() or
+ * rend_service_allow_non_anonymous_connection() whenever possible, so that
+ * checks are specific to Single Onion Services or Tor2web. */
+int
+rend_allow_non_anonymous_connection(const or_options_t* options)
+{
+  return (rend_client_allow_non_anonymous_connection(options)
+          || rend_service_allow_non_anonymous_connection(options));
+}
+
+/* Is this a rend client or server in non-anonymous mode?
+ * Clients must be specifically compiled in this mode.
+ * Onion services can be configured to start in this mode.
+ * Prefer rend_client_non_anonymous_mode_enabled() or
+ * rend_service_non_anonymous_mode_enabled() whenever possible, so that checks
+ * are specific to Single Onion Services or Tor2web. */
+int
+rend_non_anonymous_mode_enabled(const or_options_t *options)
+{
+  return (rend_client_non_anonymous_mode_enabled(options)
+          || rend_service_non_anonymous_mode_enabled(options));
+}
+
+/* Make sure that tor only builds one-hop circuits when they would not
+ * compromise user anonymity.
+ *
+ * One-hop circuits are permitted in Tor2web or Single Onion modes.
+ *
+ * Tor2web or Single Onion modes are also allowed to make multi-hop circuits.
+ * For example, single onion HSDir circuits are 3-hop to prevent denial of
+ * service.
+ */
+void
+assert_circ_anonymity_ok(const origin_circuit_t *circ,
+                         const or_options_t *options)
+{
+  tor_assert(options);
+  tor_assert(circ);
+  tor_assert(circ->build_state);
+
+  if (circ->build_state->onehop_tunnel) {
+    tor_assert(rend_allow_non_anonymous_connection(options));
+  }
+}
+
+/* Return 1 iff the given <b>digest</b> of a permenanent hidden service key is
+ * equal to the digest in the origin circuit <b>ocirc</b> of its rend data .
+ * If the rend data doesn't exist, 0 is returned. This function is agnostic to
+ * the rend data version. */
+int
+rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc,
+                          const uint8_t *digest)
+{
+  size_t rend_pk_digest_len;
+  const uint8_t *rend_pk_digest;
+
+  tor_assert(ocirc);
+  tor_assert(digest);
+
+  if (ocirc->rend_data == NULL) {
+    goto no_match;
+  }
+
+  rend_pk_digest = rend_data_get_pk_digest(ocirc->rend_data,
+                                           &rend_pk_digest_len);
+  if (tor_memeq(rend_pk_digest, digest, rend_pk_digest_len)) {
+    goto match;
+  }
+ no_match:
+  return 0;
+ match:
+  return 1;
+}
diff --git a/src/feature/rend/rendcommon.h b/src/feature/rend/rendcommon.h
new file mode 100644
index 0000000000..4ea35f88c2
--- /dev/null
+++ b/src/feature/rend/rendcommon.h
@@ -0,0 +1,82 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendcommon.h
+ * \brief Header file for rendcommon.c.
+ **/
+
+#ifndef TOR_RENDCOMMON_H
+#define TOR_RENDCOMMON_H
+
+typedef enum rend_intro_point_failure_t {
+  INTRO_POINT_FAILURE_GENERIC     = 0,
+  INTRO_POINT_FAILURE_TIMEOUT     = 1,
+  INTRO_POINT_FAILURE_UNREACHABLE = 2,
+} rend_intro_point_failure_t;
+
+int rend_cmp_service_ids(const char *one, const char *two);
+
+void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
+                             int command, size_t length,
+                             const uint8_t *payload);
+
+void rend_service_descriptor_free_(rend_service_descriptor_t *desc);
+#define rend_service_descriptor_free(desc) \
+  FREE_AND_NULL(rend_service_descriptor_t, rend_service_descriptor_free_, \
+                (desc))
+int rend_get_service_id(crypto_pk_t *pk, char *out);
+void rend_encoded_v2_service_descriptor_free_(
+                               rend_encoded_v2_service_descriptor_t *desc);
+#define rend_encoded_v2_service_descriptor_free(desc) \
+  FREE_AND_NULL(rend_encoded_v2_service_descriptor_t, \
+                rend_encoded_v2_service_descriptor_free_, (desc))
+void rend_intro_point_free_(rend_intro_point_t *intro);
+#define rend_intro_point_free(intro) \
+  FREE_AND_NULL(rend_intro_point_t, rend_intro_point_free_, (intro))
+
+int rend_valid_v2_service_id(const char *query);
+int rend_valid_descriptor_id(const char *query);
+int rend_valid_client_name(const char *client_name);
+int rend_encode_v2_descriptors(smartlist_t *descs_out,
+                               rend_service_descriptor_t *desc, time_t now,
+                               uint8_t period, rend_auth_type_t auth_type,
+                               crypto_pk_t *client_key,
+                               smartlist_t *client_cookies);
+int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id,
+                            const char *descriptor_cookie,
+                            time_t now, uint8_t replica);
+void rend_get_descriptor_id_bytes(char *descriptor_id_out,
+                                  const char *service_id,
+                                  const char *secret_id_part);
+int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs,
+                                         const char *id);
+
+int rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc,
+                              const uint8_t *digest);
+
+char *rend_auth_encode_cookie(const uint8_t *cookie_in,
+                              rend_auth_type_t auth_type);
+int rend_auth_decode_cookie(const char *cookie_in,
+                            uint8_t *cookie_out,
+                            rend_auth_type_t *auth_type_out,
+                            char **err_msg_out);
+
+int rend_allow_non_anonymous_connection(const or_options_t* options);
+int rend_non_anonymous_mode_enabled(const or_options_t *options);
+
+void assert_circ_anonymity_ok(const origin_circuit_t *circ,
+                              const or_options_t *options);
+
+#ifdef RENDCOMMON_PRIVATE
+
+STATIC int
+rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc);
+
+#endif /* defined(RENDCOMMON_PRIVATE) */
+
+#endif /* !defined(TOR_RENDCOMMON_H) */
+
diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c
new file mode 100644
index 0000000000..38c1c52e43
--- /dev/null
+++ b/src/feature/rend/rendmid.c
@@ -0,0 +1,370 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendmid.c
+ * \brief Implement introductions points and rendezvous points.
+ **/
+
+#include "or/or.h"
+#include "or/channel.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto.h"
+#include "or/dos.h"
+#include "or/relay.h"
+#include "or/rendmid.h"
+#include "or/rephist.h"
+#include "or/hs_circuitmap.h"
+#include "or/hs_intropoint.h"
+
+#include "or/or_circuit_st.h"
+
+/** Respond to an ESTABLISH_INTRO cell by checking the signed data and
+ * setting the circuit's purpose and service pk digest.
+ */
+int
+rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
+                                size_t request_len)
+{
+  crypto_pk_t *pk = NULL;
+  char buf[DIGEST_LEN+9];
+  char expected_digest[DIGEST_LEN];
+  char pk_digest[DIGEST_LEN];
+  size_t asn1len;
+  or_circuit_t *c;
+  char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  int reason = END_CIRC_REASON_INTERNAL;
+
+  log_info(LD_REND,
+           "Received a legacy ESTABLISH_INTRO request on circuit %u",
+           (unsigned) circ->p_circ_id);
+
+  if (!hs_intro_circuit_is_suitable_for_establish_intro(circ)) {
+    reason = END_CIRC_REASON_TORPROTOCOL;
+    goto err;
+  }
+
+  if (request_len < 2+DIGEST_LEN)
+    goto truncated;
+  /* First 2 bytes: length of asn1-encoded key. */
+  asn1len = ntohs(get_uint16(request));
+
+  /* Next asn1len bytes: asn1-encoded key. */
+  if (request_len < 2+DIGEST_LEN+asn1len)
+    goto truncated;
+  pk = crypto_pk_asn1_decode((char*)(request+2), asn1len);
+  if (!pk) {
+    reason = END_CIRC_REASON_TORPROTOCOL;
+    log_warn(LD_PROTOCOL, "Couldn't decode public key.");
+    goto err;
+  }
+
+  /* Next 20 bytes: Hash of rend_circ_nonce | "INTRODUCE" */
+  memcpy(buf, circ->rend_circ_nonce, DIGEST_LEN);
+  memcpy(buf+DIGEST_LEN, "INTRODUCE", 9);
+  if (crypto_digest(expected_digest, buf, DIGEST_LEN+9) < 0) {
+    log_warn(LD_BUG, "Internal error computing digest.");
+    goto err;
+  }
+  if (tor_memneq(expected_digest, request+2+asn1len, DIGEST_LEN)) {
+    log_warn(LD_PROTOCOL, "Hash of session info was not as expected.");
+    reason = END_CIRC_REASON_TORPROTOCOL;
+    goto err;
+  }
+  /* Rest of body: signature of previous data */
+  if (crypto_pk_public_checksig_digest(pk,
+                                       (char*)request, 2+asn1len+DIGEST_LEN,
+                                       (char*)(request+2+DIGEST_LEN+asn1len),
+                                       request_len-(2+DIGEST_LEN+asn1len))<0) {
+    log_warn(LD_PROTOCOL,
+             "Incorrect signature on ESTABLISH_INTRO cell; rejecting.");
+    reason = END_CIRC_REASON_TORPROTOCOL;
+    goto err;
+  }
+
+  /* The request is valid.  First, compute the hash of the service's PK.*/
+  if (crypto_pk_get_digest(pk, pk_digest)<0) {
+    log_warn(LD_BUG, "Internal error: couldn't hash public key.");
+    goto err;
+  }
+
+  crypto_pk_free(pk); /* don't need it anymore */
+  pk = NULL; /* so we don't free it again if err */
+
+  base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                pk_digest, REND_SERVICE_ID_LEN);
+
+  /* Close any other intro circuits with the same pk. */
+  c = NULL;
+  while ((c = hs_circuitmap_get_intro_circ_v2_relay_side(
+                                                (const uint8_t *)pk_digest))) {
+    log_info(LD_REND, "Replacing old circuit for service %s",
+             safe_str(serviceid));
+    circuit_mark_for_close(TO_CIRCUIT(c), END_CIRC_REASON_FINISHED);
+    /* Now it's marked, and it won't be returned next time. */
+  }
+
+  /* Acknowledge the request. */
+  if (hs_intro_send_intro_established_cell(circ) < 0) {
+    log_info(LD_GENERAL, "Couldn't send INTRO_ESTABLISHED cell.");
+    goto err_no_close;
+  }
+
+  /* Now, set up this circuit. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
+  hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest);
+
+  log_info(LD_REND,
+           "Established introduction point on circuit %u for service %s",
+           (unsigned) circ->p_circ_id, safe_str(serviceid));
+
+  return 0;
+ truncated:
+  log_warn(LD_PROTOCOL, "Rejecting truncated ESTABLISH_INTRO cell.");
+  reason = END_CIRC_REASON_TORPROTOCOL;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), reason);
+ err_no_close:
+  if (pk) crypto_pk_free(pk);
+  return -1;
+}
+
+/** Process an INTRODUCE1 cell by finding the corresponding introduction
+ * circuit, and relaying the body of the INTRODUCE1 cell inside an
+ * INTRODUCE2 cell.
+ */
+int
+rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
+                          size_t request_len)
+{
+  or_circuit_t *intro_circ;
+  char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  char nak_body[1];
+
+  log_info(LD_REND, "Received an INTRODUCE1 request on circuit %u",
+           (unsigned)circ->p_circ_id);
+
+  /* At this point, we know that the circuit is valid for an INTRODUCE1
+   * because the validation has been made before calling this function. */
+  tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_OR);
+  tor_assert(!circ->base_.n_chan);
+
+  /* We could change this to MAX_HEX_NICKNAME_LEN now that 0.0.9.x is
+   * obsolete; however, there isn't much reason to do so, and we're going
+   * to revise this protocol anyway.
+   */
+  if (request_len < (DIGEST_LEN+(MAX_NICKNAME_LEN+1)+REND_COOKIE_LEN+
+                     DH1024_KEY_LEN+CIPHER_KEY_LEN+
+                     PKCS1_OAEP_PADDING_OVERHEAD)) {
+    log_warn(LD_PROTOCOL, "Impossibly short INTRODUCE1 cell on circuit %u; "
+             "responding with nack.",
+             (unsigned)circ->p_circ_id);
+    goto err;
+  }
+
+  base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                (char*)request, REND_SERVICE_ID_LEN);
+
+  /* The first 20 bytes are all we look at: they have a hash of the service's
+   * PK. */
+  intro_circ = hs_circuitmap_get_intro_circ_v2_relay_side(
+                                                      (const uint8_t*)request);
+  if (!intro_circ) {
+    log_info(LD_REND,
+             "No intro circ found for INTRODUCE1 cell (%s) from circuit %u; "
+             "responding with nack.",
+             safe_str(serviceid), (unsigned)circ->p_circ_id);
+    goto err;
+  }
+
+  log_info(LD_REND,
+           "Sending introduction request for service %s "
+           "from circ %u to circ %u",
+           safe_str(serviceid), (unsigned)circ->p_circ_id,
+           (unsigned)intro_circ->p_circ_id);
+
+  /* Great.  Now we just relay the cell down the circuit. */
+  if (relay_send_command_from_edge(0, TO_CIRCUIT(intro_circ),
+                                   RELAY_COMMAND_INTRODUCE2,
+                                   (char*)request, request_len, NULL)) {
+    log_warn(LD_GENERAL,
+             "Unable to send INTRODUCE2 cell to Tor client.");
+    /* Stop right now, the circuit has been closed. */
+    return -1;
+  }
+  /* And send an ack down the client's circuit.  Empty body means succeeded. */
+  if (relay_send_command_from_edge(0,TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_INTRODUCE_ACK,
+                                   NULL,0,NULL)) {
+    log_warn(LD_GENERAL, "Unable to send INTRODUCE_ACK cell to Tor client.");
+    /* Stop right now, the circuit has been closed. */
+    return -1;
+  }
+
+  return 0;
+ err:
+  /* Send the client a NACK */
+  nak_body[0] = 1;
+  if (relay_send_command_from_edge(0,TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_INTRODUCE_ACK,
+                                   nak_body, 1, NULL)) {
+    log_warn(LD_GENERAL, "Unable to send NAK to Tor client.");
+  }
+  return -1;
+}
+
+/** Process an ESTABLISH_RENDEZVOUS cell by setting the circuit's purpose and
+ * rendezvous cookie.
+ */
+int
+rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
+                              size_t request_len)
+{
+  char hexid[9];
+  int reason = END_CIRC_REASON_TORPROTOCOL;
+
+  log_info(LD_REND, "Received an ESTABLISH_RENDEZVOUS request on circuit %u",
+           (unsigned)circ->p_circ_id);
+
+  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR) {
+    log_warn(LD_PROTOCOL,
+             "Tried to establish rendezvous on non-OR circuit with purpose %s",
+             circuit_purpose_to_string(circ->base_.purpose));
+    goto err;
+  }
+
+  /* Check if we are configured to accept established rendezvous cells from
+   * client or in other words tor2web clients. */
+  if (channel_is_client(circ->p_chan) &&
+      dos_should_refuse_single_hop_client()) {
+    /* Note it down for the heartbeat log purposes. */
+    dos_note_refuse_single_hop_client();
+    /* Silent drop so the client has to time out before moving on. */
+    return 0;
+  }
+
+  if (circ->base_.n_chan) {
+    log_warn(LD_PROTOCOL,
+             "Tried to establish rendezvous on non-edge circuit");
+    goto err;
+  }
+
+  if (request_len != REND_COOKIE_LEN) {
+    log_fn(LOG_PROTOCOL_WARN,
+           LD_PROTOCOL, "Invalid length on ESTABLISH_RENDEZVOUS.");
+    goto err;
+  }
+
+  if (hs_circuitmap_get_rend_circ_relay_side(request)) {
+    log_warn(LD_PROTOCOL,
+             "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS.");
+    goto err;
+  }
+
+  /* Acknowledge the request. */
+  if (relay_send_command_from_edge(0,TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_RENDEZVOUS_ESTABLISHED,
+                                   "", 0, NULL)<0) {
+    log_warn(LD_PROTOCOL, "Couldn't send RENDEZVOUS_ESTABLISHED cell.");
+    /* Stop right now, the circuit has been closed. */
+    return -1;
+  }
+
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_POINT_WAITING);
+  hs_circuitmap_register_rend_circ_relay_side(circ, request);
+
+  base16_encode(hexid,9,(char*)request,4);
+
+  log_info(LD_REND,
+           "Established rendezvous point on circuit %u for cookie %s",
+           (unsigned)circ->p_circ_id, hexid);
+
+  return 0;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), reason);
+  return -1;
+}
+
+/** Process a RENDEZVOUS1 cell by looking up the correct rendezvous
+ * circuit by its relaying the cell's body in a RENDEZVOUS2 cell, and
+ * connecting the two circuits.
+ */
+int
+rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
+                    size_t request_len)
+{
+  const or_options_t *options = get_options();
+  or_circuit_t *rend_circ;
+  char hexid[9];
+  int reason = END_CIRC_REASON_INTERNAL;
+
+  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) {
+    log_info(LD_REND,
+             "Tried to complete rendezvous on non-OR or non-edge circuit %u.",
+             (unsigned)circ->p_circ_id);
+    reason = END_CIRC_REASON_TORPROTOCOL;
+    goto err;
+  }
+
+  if (request_len < REND_COOKIE_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+         "Rejecting RENDEZVOUS1 cell with bad length (%d) on circuit %u.",
+         (int)request_len, (unsigned)circ->p_circ_id);
+    reason = END_CIRC_REASON_TORPROTOCOL;
+    goto err;
+  }
+
+  base16_encode(hexid, sizeof(hexid), (const char*)request, 4);
+
+  log_info(LD_REND,
+           "Got request for rendezvous from circuit %u to cookie %s.",
+           (unsigned)circ->p_circ_id, hexid);
+
+  rend_circ = hs_circuitmap_get_rend_circ_relay_side(request);
+  if (!rend_circ) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+         "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.",
+         hexid);
+    reason = END_CIRC_REASON_TORPROTOCOL;
+    goto err;
+  }
+
+  /* Statistics: Mark this circuit as an RP circuit so that we collect
+     stats from it. */
+  if (options->HiddenServiceStatistics) {
+    circ->circuit_carries_hs_traffic_stats = 1;
+  }
+
+  /* Send the RENDEZVOUS2 cell to the client. */
+  if (relay_send_command_from_edge(0, TO_CIRCUIT(rend_circ),
+                                   RELAY_COMMAND_RENDEZVOUS2,
+                                   (char*)(request+REND_COOKIE_LEN),
+                                   request_len-REND_COOKIE_LEN, NULL)) {
+    log_warn(LD_GENERAL,
+             "Unable to send RENDEZVOUS2 cell to client on circuit %u.",
+             (unsigned)rend_circ->p_circ_id);
+    /* Stop right now, the circuit has been closed. */
+    return -1;
+  }
+
+  /* Join the circuits. */
+  log_info(LD_REND,
+           "Completing rendezvous: circuit %u joins circuit %u (cookie %s)",
+           (unsigned)circ->p_circ_id, (unsigned)rend_circ->p_circ_id, hexid);
+
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_ESTABLISHED);
+  circuit_change_purpose(TO_CIRCUIT(rend_circ),
+                         CIRCUIT_PURPOSE_REND_ESTABLISHED);
+  hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
+
+  rend_circ->rend_splice = circ;
+  circ->rend_splice = rend_circ;
+
+  return 0;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), reason);
+  return -1;
+}
diff --git a/src/feature/rend/rendmid.h b/src/feature/rend/rendmid.h
new file mode 100644
index 0000000000..907a0c6a73
--- /dev/null
+++ b/src/feature/rend/rendmid.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendmid.h
+ * \brief Header file for rendmid.c.
+ **/
+
+#ifndef TOR_RENDMID_H
+#define TOR_RENDMID_H
+
+int rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
+                                    size_t request_len);
+int rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
+                              size_t request_len);
+int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
+                                  size_t request_len);
+int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
+                        size_t request_len);
+
+#endif /* !defined(TOR_RENDMID_H) */
+
diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c
new file mode 100644
index 0000000000..8e094b593c
--- /dev/null
+++ b/src/feature/rend/rendservice.c
@@ -0,0 +1,4451 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendservice.c
+ * \brief The hidden-service side of rendezvous functionality.
+ **/
+
+#define RENDSERVICE_PRIVATE
+
+#include "or/or.h"
+#include "or/circpathbias.h"
+#include "or/circuitbuild.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "or/control.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "or/directory.h"
+#include "or/hs_common.h"
+#include "or/hs_config.h"
+#include "or/main.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/policies.h"
+#include "or/rendclient.h"
+#include "or/rendcommon.h"
+#include "or/rendservice.h"
+#include "or/router.h"
+#include "or/relay.h"
+#include "or/rephist.h"
+#include "or/replaycache.h"
+#include "or/routerlist.h"
+#include "or/routerparse.h"
+#include "or/routerset.h"
+#include "lib/encoding/confline.h"
+
+#include "or/cpath_build_state_st.h"
+#include "or/crypt_path_st.h"
+#include "or/crypt_path_reference_st.h"
+#include "or/edge_connection_st.h"
+#include "or/extend_info_st.h"
+#include "or/networkstatus_st.h"
+#include "or/origin_circuit_st.h"
+#include "or/rend_authorized_client_st.h"
+#include "or/rend_encoded_v2_service_descriptor_st.h"
+#include "or/rend_intro_point_st.h"
+#include "or/rend_service_descriptor_st.h"
+#include "or/routerstatus_st.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+struct rend_service_t;
+static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro,
+                                            const char *pk_digest);
+static rend_intro_point_t *find_intro_point(origin_circuit_t *circ);
+static rend_intro_point_t *find_expiring_intro_point(
+    struct rend_service_t *service, origin_circuit_t *circ);
+
+static extend_info_t *find_rp_for_intro(
+    const rend_intro_cell_t *intro,
+    char **err_msg_out);
+
+static int intro_point_accepted_intro_count(rend_intro_point_t *intro);
+static int intro_point_should_expire_now(rend_intro_point_t *intro,
+                                         time_t now);
+static int rend_service_derive_key_digests(struct rend_service_t *s);
+static int rend_service_load_keys(struct rend_service_t *s);
+static int rend_service_load_auth_keys(struct rend_service_t *s,
+                                       const char *hfname);
+static struct rend_service_t *rend_service_get_by_pk_digest(
+    const char* digest);
+static struct rend_service_t *rend_service_get_by_service_id(const char *id);
+static const char *rend_service_escaped_dir(
+    const struct rend_service_t *s);
+
+static ssize_t rend_service_parse_intro_for_v0_or_v1(
+    rend_intro_cell_t *intro,
+    const uint8_t *buf,
+    size_t plaintext_len,
+    char **err_msg_out);
+static ssize_t rend_service_parse_intro_for_v2(
+    rend_intro_cell_t *intro,
+    const uint8_t *buf,
+    size_t plaintext_len,
+    char **err_msg_out);
+static ssize_t rend_service_parse_intro_for_v3(
+    rend_intro_cell_t *intro,
+    const uint8_t *buf,
+    size_t plaintext_len,
+    char **err_msg_out);
+
+static int rend_service_check_private_dir(const or_options_t *options,
+                                          const rend_service_t *s,
+                                          int create);
+static const smartlist_t* rend_get_service_list(
+                                  const smartlist_t* substitute_service_list);
+static smartlist_t* rend_get_service_list_mutable(
+                                  smartlist_t* substitute_service_list);
+static int rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted);
+
+/* Hidden service directory file names:
+ * new file names should be added to rend_service_add_filenames_to_list()
+ * for sandboxing purposes. */
+static const char *private_key_fname = "private_key";
+static const char *hostname_fname = "hostname";
+static const char *client_keys_fname = "client_keys";
+static const char *sos_poison_fname = "onion_service_non_anonymous";
+
+/** A list of rend_service_t's for services run on this OP. */
+static smartlist_t *rend_service_list = NULL;
+/** A list of rend_service_t's for services run on this OP which is used as a
+ * staging area before they are put in the main list in order to prune dying
+ * service on config reload. */
+static smartlist_t *rend_service_staging_list = NULL;
+
+/* Like rend_get_service_list_mutable, but returns a read-only list. */
+static const smartlist_t*
+rend_get_service_list(const smartlist_t* substitute_service_list)
+{
+  /* It is safe to cast away the const here, because
+   * rend_get_service_list_mutable does not actually modify the list */
+  return rend_get_service_list_mutable((smartlist_t*)substitute_service_list);
+}
+
+/* Return a mutable list of hidden services.
+ * If substitute_service_list is not NULL, return it.
+ * Otherwise, check if the global rend_service_list is non-NULL, and if so,
+ * return it.
+ * Otherwise, log a BUG message and return NULL.
+ * */
+static smartlist_t*
+rend_get_service_list_mutable(smartlist_t* substitute_service_list)
+{
+  if (substitute_service_list) {
+    return substitute_service_list;
+  }
+
+  /* If no special service list is provided, then just use the global one. */
+
+  if (BUG(!rend_service_list)) {
+    /* No global HS list, which is a programmer error. */
+    return NULL;
+  }
+
+  return rend_service_list;
+}
+
+/** Tells if onion service <b>s</b> is ephemeral.
+ */
+static unsigned int
+rend_service_is_ephemeral(const struct rend_service_t *s)
+{
+  return (s->directory == NULL);
+}
+
+/** Returns a escaped string representation of the service, <b>s</b>.
+ */
+static const char *
+rend_service_escaped_dir(const struct rend_service_t *s)
+{
+  return rend_service_is_ephemeral(s) ? "[EPHEMERAL]" : escaped(s->directory);
+}
+
+/** Return the number of rendezvous services we have configured. */
+int
+rend_num_services(void)
+{
+  if (!rend_service_list)
+    return 0;
+  return smartlist_len(rend_service_list);
+}
+
+/** Helper: free storage held by a single service authorized client entry. */
+void
+rend_authorized_client_free_(rend_authorized_client_t *client)
+{
+  if (!client)
+    return;
+  if (client->client_key)
+    crypto_pk_free(client->client_key);
+  if (client->client_name)
+    memwipe(client->client_name, 0, strlen(client->client_name));
+  tor_free(client->client_name);
+  memwipe(client->descriptor_cookie, 0, sizeof(client->descriptor_cookie));
+  tor_free(client);
+}
+
+/** Helper for strmap_free. */
+static void
+rend_authorized_client_free_void(void *authorized_client)
+{
+  rend_authorized_client_free_(authorized_client);
+}
+
+/** Release the storage held by <b>service</b>.
+ */
+STATIC void
+rend_service_free_(rend_service_t *service)
+{
+  if (!service)
+    return;
+
+  tor_free(service->directory);
+  if (service->ports) {
+    SMARTLIST_FOREACH(service->ports, rend_service_port_config_t*, p,
+                      rend_service_port_config_free(p));
+    smartlist_free(service->ports);
+  }
+  if (service->private_key)
+    crypto_pk_free(service->private_key);
+  if (service->intro_nodes) {
+    SMARTLIST_FOREACH(service->intro_nodes, rend_intro_point_t *, intro,
+      rend_intro_point_free(intro););
+    smartlist_free(service->intro_nodes);
+  }
+  if (service->expiring_nodes) {
+    SMARTLIST_FOREACH(service->expiring_nodes, rend_intro_point_t *, intro,
+                      rend_intro_point_free(intro););
+    smartlist_free(service->expiring_nodes);
+  }
+
+  rend_service_descriptor_free(service->desc);
+  if (service->clients) {
+    SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, c,
+      rend_authorized_client_free(c););
+    smartlist_free(service->clients);
+  }
+  if (service->accepted_intro_dh_parts) {
+    replaycache_free(service->accepted_intro_dh_parts);
+  }
+  tor_free(service);
+}
+
+/* Release all the storage held in rend_service_staging_list. */
+void
+rend_service_free_staging_list(void)
+{
+  if (rend_service_staging_list) {
+    SMARTLIST_FOREACH(rend_service_staging_list, rend_service_t*, ptr,
+                      rend_service_free(ptr));
+    smartlist_free(rend_service_staging_list);
+    rend_service_staging_list = NULL;
+  }
+}
+
+/** Release all the storage held in both rend_service_list and
+ * rend_service_staging_list. */
+void
+rend_service_free_all(void)
+{
+  if (rend_service_list) {
+    SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr,
+                      rend_service_free(ptr));
+    smartlist_free(rend_service_list);
+    rend_service_list = NULL;
+  }
+  rend_service_free_staging_list();
+}
+
+/* Initialize the subsystem. */
+void
+rend_service_init(void)
+{
+  tor_assert(!rend_service_list);
+  tor_assert(!rend_service_staging_list);
+
+  rend_service_list = smartlist_new();
+  rend_service_staging_list = smartlist_new();
+}
+
+/* Validate a <b>service</b>. Use the <b>service_list</b> to make sure there
+ * is no duplicate entry for the given service object. Return 0 if valid else
+ * -1 if not.*/
+static int
+rend_validate_service(const smartlist_t *service_list,
+                      const rend_service_t *service)
+{
+  tor_assert(service_list);
+  tor_assert(service);
+
+  if (service->max_streams_per_circuit < 0) {
+    log_warn(LD_CONFIG, "Hidden service (%s) configured with negative max "
+                        "streams per circuit.",
+             rend_service_escaped_dir(service));
+    goto invalid;
+  }
+
+  if (service->max_streams_close_circuit < 0 ||
+      service->max_streams_close_circuit > 1) {
+    log_warn(LD_CONFIG, "Hidden service (%s) configured with invalid "
+                        "max streams handling.",
+             rend_service_escaped_dir(service));
+    goto invalid;
+  }
+
+  if (service->auth_type != REND_NO_AUTH &&
+      (!service->clients || smartlist_len(service->clients) == 0)) {
+    log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but "
+                        "no clients.",
+             rend_service_escaped_dir(service));
+    goto invalid;
+  }
+
+  if (!service->ports || !smartlist_len(service->ports)) {
+    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured.",
+             rend_service_escaped_dir(service));
+    goto invalid;
+  }
+
+  /* Valid. */
+  return 0;
+ invalid:
+  return -1;
+}
+
+/** Add it to <b>service_list</b>, or to the global rend_service_list if
+ * <b>service_list</b> is NULL.  Return 0 on success.  On failure, free
+ * <b>service</b> and return -1. Takes ownership of <b>service</b>. */
+static int
+rend_add_service(smartlist_t *service_list, rend_service_t *service)
+{
+  int i;
+  rend_service_port_config_t *p;
+
+  tor_assert(service);
+
+  smartlist_t *s_list = rend_get_service_list_mutable(service_list);
+  /* We must have a service list, even if it's a temporary one, so we can
+   * check for duplicate services */
+  if (BUG(!s_list)) {
+    rend_service_free(service);
+    return -1;
+  }
+
+  service->intro_nodes = smartlist_new();
+  service->expiring_nodes = smartlist_new();
+
+  log_debug(LD_REND,"Configuring service with directory %s",
+            rend_service_escaped_dir(service));
+  for (i = 0; i < smartlist_len(service->ports); ++i) {
+    p = smartlist_get(service->ports, i);
+    if (!(p->is_unix_addr)) {
+      log_debug(LD_REND,
+                "Service maps port %d to %s",
+                p->virtual_port,
+                fmt_addrport(&p->real_addr, p->real_port));
+    } else {
+#ifdef HAVE_SYS_UN_H
+      log_debug(LD_REND,
+                "Service maps port %d to socket at \"%s\"",
+                p->virtual_port, p->unix_addr);
+#else
+      log_warn(LD_BUG,
+               "Service maps port %d to an AF_UNIX socket, but we "
+               "have no AF_UNIX support on this platform.  This is "
+               "probably a bug.",
+               p->virtual_port);
+      rend_service_free(service);
+      return -1;
+#endif /* defined(HAVE_SYS_UN_H) */
+    }
+  }
+  /* The service passed all the checks */
+  tor_assert(s_list);
+  smartlist_add(s_list, service);
+
+  /* Notify that our global service list has changed only if this new service
+   * went into our global list. If not, when we move service from the staging
+   * list to the new list, a notify is triggered. */
+  if (s_list == rend_service_list) {
+    hs_service_map_has_changed();
+  }
+  return 0;
+}
+
+/** Return a new rend_service_port_config_t with its path set to
+ * <b>socket_path</b> or empty if <b>socket_path</b> is NULL */
+static rend_service_port_config_t *
+rend_service_port_config_new(const char *socket_path)
+{
+  if (!socket_path)
+    return tor_malloc_zero(sizeof(rend_service_port_config_t) + 1);
+
+  const size_t pathlen = strlen(socket_path) + 1;
+  rend_service_port_config_t *conf =
+    tor_malloc_zero(sizeof(rend_service_port_config_t) + pathlen);
+  memcpy(conf->unix_addr, socket_path, pathlen);
+  conf->is_unix_addr = 1;
+  return conf;
+}
+
+/** Parses a virtual-port to real-port/socket mapping separated by
+ * the provided separator and returns a new rend_service_port_config_t,
+ * or NULL and an optional error string on failure.
+ *
+ * The format is: VirtualPort SEP (IP|RealPort|IP:RealPort|'socket':path)?
+ *
+ * IP defaults to 127.0.0.1; RealPort defaults to VirtualPort.
+ */
+rend_service_port_config_t *
+rend_service_parse_port_config(const char *string, const char *sep,
+                               char **err_msg_out)
+{
+  smartlist_t *sl;
+  int virtport;
+  int realport = 0;
+  uint16_t p;
+  tor_addr_t addr;
+  rend_service_port_config_t *result = NULL;
+  unsigned int is_unix_addr = 0;
+  const char *socket_path = NULL;
+  char *err_msg = NULL;
+  char *addrport = NULL;
+
+  sl = smartlist_new();
+  smartlist_split_string(sl, string, sep,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2);
+  if (smartlist_len(sl) < 1 || BUG(smartlist_len(sl) > 2)) {
+    err_msg = tor_strdup("Bad syntax in hidden service port configuration.");
+    goto err;
+  }
+  virtport = (int)tor_parse_long(smartlist_get(sl,0), 10, 1, 65535, NULL,NULL);
+  if (!virtport) {
+    tor_asprintf(&err_msg, "Missing or invalid port %s in hidden service "
+                   "port configuration", escaped(smartlist_get(sl,0)));
+
+    goto err;
+  }
+  if (smartlist_len(sl) == 1) {
+    /* No addr:port part; use default. */
+    realport = virtport;
+    tor_addr_from_ipv4h(&addr, 0x7F000001u); /* 127.0.0.1 */
+  } else {
+    int ret;
+
+    const char *addrport_element = smartlist_get(sl,1);
+    const char *rest = NULL;
+    int is_unix;
+    ret = port_cfg_line_extract_addrport(addrport_element, &addrport,
+                                         &is_unix, &rest);
+    if (ret < 0) {
+      tor_asprintf(&err_msg, "Couldn't process address <%s> from hidden "
+                   "service configuration", addrport_element);
+      goto err;
+    }
+    if (is_unix) {
+      socket_path = addrport;
+      is_unix_addr = 1;
+    } else if (strchr(addrport, ':') || strchr(addrport, '.')) {
+      /* else try it as an IP:port pair if it has a : or . in it */
+      if (tor_addr_port_lookup(addrport, &addr, &p)<0) {
+        err_msg = tor_strdup("Unparseable address in hidden service port "
+                             "configuration.");
+        goto err;
+      }
+      realport = p?p:virtport;
+    } else {
+      /* No addr:port, no addr -- must be port. */
+      realport = (int)tor_parse_long(addrport, 10, 1, 65535, NULL, NULL);
+      if (!realport) {
+        tor_asprintf(&err_msg, "Unparseable or out-of-range port %s in "
+                     "hidden service port configuration.",
+                     escaped(addrport));
+        goto err;
+      }
+      tor_addr_from_ipv4h(&addr, 0x7F000001u); /* Default to 127.0.0.1 */
+    }
+  }
+
+  /* Allow room for unix_addr */
+  result = rend_service_port_config_new(socket_path);
+  result->virtual_port = virtport;
+  result->is_unix_addr = is_unix_addr;
+  if (!is_unix_addr) {
+    result->real_port = realport;
+    tor_addr_copy(&result->real_addr, &addr);
+    result->unix_addr[0] = '\0';
+  }
+
+ err:
+  tor_free(addrport);
+  if (err_msg_out != NULL) {
+    *err_msg_out = err_msg;
+  } else {
+    tor_free(err_msg);
+  }
+  SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+  smartlist_free(sl);
+
+  return result;
+}
+
+/** Release all storage held in a rend_service_port_config_t. */
+void
+rend_service_port_config_free_(rend_service_port_config_t *p)
+{
+  tor_free(p);
+}
+
+/* Copy relevant data from service src to dst while pruning the service lists.
+ * This should only be called during the pruning process which takes existing
+ * services and copy their data to the newly configured services. The src
+ * service replaycache will be set to NULL after this call. */
+static void
+copy_service_on_prunning(rend_service_t *dst, rend_service_t *src)
+{
+  tor_assert(dst);
+  tor_assert(src);
+
+  /* Keep the timestamps for when the content changed and the next upload
+   * time so we can properly upload the descriptor if needed for the new
+   * service object. */
+  dst->desc_is_dirty = src->desc_is_dirty;
+  dst->next_upload_time = src->next_upload_time;
+  /* Move the replaycache to the new object. */
+  dst->accepted_intro_dh_parts = src->accepted_intro_dh_parts;
+  src->accepted_intro_dh_parts = NULL;
+  /* Copy intro point information to destination service. */
+  dst->intro_period_started = src->intro_period_started;
+  dst->n_intro_circuits_launched = src->n_intro_circuits_launched;
+  dst->n_intro_points_wanted = src->n_intro_points_wanted;
+}
+
+/* Helper: Actual implementation of the pruning on reload which we've
+ * decoupled in order to make the unit test workeable without ugly hacks.
+ * Furthermore, this function does NOT free any memory but will nullify the
+ * temporary list pointer whatever happens. */
+STATIC void
+rend_service_prune_list_impl_(void)
+{
+  origin_circuit_t *ocirc = NULL;
+  smartlist_t *surviving_services, *old_service_list, *new_service_list;
+
+  /* When pruning our current service list, we must have a staging list that
+   * contains what we want to check else it's a code flow error. */
+  tor_assert(rend_service_staging_list);
+
+  /* We are about to prune the current list of its dead service so set the
+   * semantic for that list to be the "old" one. */
+  old_service_list = rend_service_list;
+  /* The staging list is now the "new" list so set this semantic. */
+  new_service_list = rend_service_staging_list;
+  /* After this, whatever happens, we'll use our new list. */
+  rend_service_list = new_service_list;
+  /* Finally, nullify the staging list pointer as we don't need it anymore
+   * and it needs to be NULL before the next reload. */
+  rend_service_staging_list = NULL;
+  /* Nothing to prune if we have no service list so stop right away. */
+  if (!old_service_list) {
+    return;
+  }
+
+  /* This contains all _existing_ services that survives the relaod that is
+   * that haven't been removed from the configuration. The difference between
+   * this list and the new service list is that the new list can possibly
+   * contain newly configured service that have no introduction points opened
+   * yet nor key material loaded or generated. */
+  surviving_services = smartlist_new();
+
+  /* Preserve the existing ephemeral services.
+   *
+   * This is the ephemeral service equivalent of the "Copy introduction
+   * points to new services" block, except there's no copy required since
+   * the service structure isn't regenerated.
+   *
+   * After this is done, all ephemeral services will be:
+   *  * Removed from old_service_list, so the equivalent non-ephemeral code
+   *    will not attempt to preserve them.
+   *  * Added to the new_service_list (that previously only had the
+   *    services listed in the configuration).
+   *  * Added to surviving_services, which is the list of services that
+   *    will NOT have their intro point closed.
+   */
+  SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
+    if (rend_service_is_ephemeral(old)) {
+      SMARTLIST_DEL_CURRENT(old_service_list, old);
+      smartlist_add(surviving_services, old);
+      smartlist_add(new_service_list, old);
+    }
+  } SMARTLIST_FOREACH_END(old);
+
+  /* Copy introduction points to new services. This is O(n^2), but it's only
+   * called on reconfigure, so it's ok performance wise. */
+  SMARTLIST_FOREACH_BEGIN(new_service_list, rend_service_t *, new) {
+    SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
+      /* Skip ephemeral services as we only want to copy introduction points
+       * from current services to newly configured one that already exists.
+       * The same directory means it's the same service. */
+      if (rend_service_is_ephemeral(new) || rend_service_is_ephemeral(old) ||
+          strcmp(old->directory, new->directory)) {
+        continue;
+      }
+      smartlist_add_all(new->intro_nodes, old->intro_nodes);
+      smartlist_clear(old->intro_nodes);
+      smartlist_add_all(new->expiring_nodes, old->expiring_nodes);
+      smartlist_clear(old->expiring_nodes);
+
+      /* Copy needed information from old to new. */
+      copy_service_on_prunning(new, old);
+
+      /* This regular service will survive the closing IPs step after. */
+      smartlist_add(surviving_services, old);
+      break;
+    } SMARTLIST_FOREACH_END(old);
+  } SMARTLIST_FOREACH_END(new);
+
+  /* For every service introduction circuit we can find, see if we have a
+   * matching surviving configured service. If not, close the circuit. */
+  while ((ocirc = circuit_get_next_service_intro_circ(ocirc))) {
+    int keep_it = 0;
+    if (ocirc->rend_data == NULL) {
+      /* This is a v3 circuit, ignore it. */
+      continue;
+    }
+    SMARTLIST_FOREACH_BEGIN(surviving_services, const rend_service_t *, s) {
+      if (rend_circuit_pk_digest_eq(ocirc, (uint8_t *) s->pk_digest)) {
+        /* Keep this circuit as we have a matching configured service. */
+        keep_it = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(s);
+    if (keep_it) {
+      continue;
+    }
+    log_info(LD_REND, "Closing intro point %s for service %s.",
+             safe_str_client(extend_info_describe(
+                                        ocirc->build_state->chosen_exit)),
+             safe_str_client(rend_data_get_address(ocirc->rend_data)));
+    /* Reason is FINISHED because service has been removed and thus the
+     * circuit is considered old/uneeded. */
+    circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+  }
+  smartlist_free(surviving_services);
+  /* Notify that our global service list has changed. */
+  hs_service_map_has_changed();
+}
+
+/* Try to prune our main service list using the temporary one that we just
+ * loaded and parsed successfully. The pruning process decides which onion
+ * services to keep and which to discard after a reload. */
+void
+rend_service_prune_list(void)
+{
+  smartlist_t *old_service_list = rend_service_list;
+
+  if (!rend_service_staging_list) {
+    rend_service_staging_list = smartlist_new();
+  }
+
+  rend_service_prune_list_impl_();
+  if (old_service_list) {
+    /* Every remaining service in the old list have been removed from the
+     * configuration so clean them up safely. */
+    SMARTLIST_FOREACH(old_service_list, rend_service_t *, s,
+                      rend_service_free(s));
+    smartlist_free(old_service_list);
+  }
+}
+
+/* Copy all the relevant data that the hs_service object contains over to the
+ * rend_service_t object. The reason to do so is because when configuring a
+ * service, we go through a generic handler that creates an hs_service_t
+ * object which so we have to copy the parsed values to a rend service object
+ * which is version 2 specific. */
+static void
+service_config_shadow_copy(rend_service_t *service,
+                           hs_service_config_t *config)
+{
+  tor_assert(service);
+  tor_assert(config);
+
+  service->directory = tor_strdup(config->directory_path);
+  service->dir_group_readable = config->dir_group_readable;
+  service->allow_unknown_ports = config->allow_unknown_ports;
+  /* This value can't go above HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT (65535)
+   * if the code flow is right so this cast is safe. But just in case, we'll
+   * check it. */
+  service->max_streams_per_circuit = (int) config->max_streams_per_rdv_circuit;
+  if (BUG(config->max_streams_per_rdv_circuit >
+          HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT)) {
+    service->max_streams_per_circuit = HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT;
+  }
+  service->max_streams_close_circuit = config->max_streams_close_circuit;
+  service->n_intro_points_wanted = config->num_intro_points;
+  /* Switching ownership of the ports to the rend service object. */
+  smartlist_add_all(service->ports, config->ports);
+  smartlist_free(config->ports);
+  config->ports = NULL;
+}
+
+/* Parse the hidden service configuration starting at <b>line_</b> using the
+ * already configured generic service configuration in <b>config</b>. This
+ * function will translate the config object to a rend_service_t and add it to
+ * the temporary list if valid. If <b>validate_only</b> is set, parse, warn
+ * and return as normal but don't actually add the service to the list. */
+int
+rend_config_service(const config_line_t *line_,
+                    const or_options_t *options,
+                    hs_service_config_t *config)
+{
+  const config_line_t *line;
+  rend_service_t *service = NULL;
+
+  /* line_ can be NULL which would mean that the service configuration only
+   * have one line that is the directory directive. */
+  tor_assert(options);
+  tor_assert(config);
+
+  /* Use the staging service list so that we can check then do the pruning
+   * process using the main list at the end. */
+  if (rend_service_staging_list == NULL) {
+    rend_service_staging_list = smartlist_new();
+  }
+
+  /* Initialize service. */
+  service = tor_malloc_zero(sizeof(rend_service_t));
+  service->intro_period_started = time(NULL);
+  service->ports = smartlist_new();
+  /* From the hs_service object which has been used to load the generic
+   * options, we'll copy over the useful data to the rend_service_t object. */
+  service_config_shadow_copy(service, config);
+
+  for (line = line_; line; line = line->next) {
+    if (!strcasecmp(line->key, "HiddenServiceDir")) {
+      /* We just hit the next hidden service, stop right now. */
+      break;
+    }
+    /* Number of introduction points. */
+    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
+      int ok = 0;
+      /* Those are specific defaults for version 2. */
+      service->n_intro_points_wanted =
+        (unsigned int) tor_parse_long(line->value, 10,
+                                      0, NUM_INTRO_POINTS_MAX, &ok, NULL);
+      if (!ok) {
+        log_warn(LD_CONFIG,
+                 "HiddenServiceNumIntroductionPoints "
+                 "should be between %d and %d, not %s",
+                 0, NUM_INTRO_POINTS_MAX, line->value);
+        goto err;
+      }
+      log_info(LD_CONFIG, "HiddenServiceNumIntroductionPoints=%d for %s",
+               service->n_intro_points_wanted, escaped(service->directory));
+      continue;
+    }
+    if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
+      /* Parse auth type and comma-separated list of client names and add a
+       * rend_authorized_client_t for each client to the service's list
+       * of authorized clients. */
+      smartlist_t *type_names_split, *clients;
+      const char *authname;
+      if (service->auth_type != REND_NO_AUTH) {
+        log_warn(LD_CONFIG, "Got multiple HiddenServiceAuthorizeClient "
+                 "lines for a single service.");
+        goto err;
+      }
+      type_names_split = smartlist_new();
+      smartlist_split_string(type_names_split, line->value, " ", 0, 2);
+      if (smartlist_len(type_names_split) < 1) {
+        log_warn(LD_BUG, "HiddenServiceAuthorizeClient has no value. This "
+                         "should have been prevented when parsing the "
+                         "configuration.");
+        smartlist_free(type_names_split);
+        goto err;
+      }
+      authname = smartlist_get(type_names_split, 0);
+      if (!strcasecmp(authname, "basic")) {
+        service->auth_type = REND_BASIC_AUTH;
+      } else if (!strcasecmp(authname, "stealth")) {
+        service->auth_type = REND_STEALTH_AUTH;
+      } else {
+        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
+                 "unrecognized auth-type '%s'. Only 'basic' or 'stealth' "
+                 "are recognized.",
+                 (char *) smartlist_get(type_names_split, 0));
+        SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
+        smartlist_free(type_names_split);
+        goto err;
+      }
+      service->clients = smartlist_new();
+      if (smartlist_len(type_names_split) < 2) {
+        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
+                            "auth-type '%s', but no client names.",
+                 service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
+        SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
+        smartlist_free(type_names_split);
+        continue;
+      }
+      clients = smartlist_new();
+      smartlist_split_string(clients, smartlist_get(type_names_split, 1),
+                             ",", SPLIT_SKIP_SPACE, 0);
+      SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
+      smartlist_free(type_names_split);
+      /* Remove duplicate client names. */
+      {
+        int num_clients = smartlist_len(clients);
+        smartlist_sort_strings(clients);
+        smartlist_uniq_strings(clients);
+        if (smartlist_len(clients) < num_clients) {
+          log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
+                   "duplicate client name(s); removing.",
+                   num_clients - smartlist_len(clients));
+        }
+      }
+      SMARTLIST_FOREACH_BEGIN(clients, const char *, client_name)
+      {
+        rend_authorized_client_t *client;
+        if (!rend_valid_client_name(client_name)) {
+          log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains an "
+                              "illegal client name: '%s'. Names must be "
+                              "between 1 and %d characters and contain "
+                              "only [A-Za-z0-9+_-].",
+                   client_name, REND_CLIENTNAME_MAX_LEN);
+          SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
+          smartlist_free(clients);
+          goto err;
+        }
+        client = tor_malloc_zero(sizeof(rend_authorized_client_t));
+        client->client_name = tor_strdup(client_name);
+        smartlist_add(service->clients, client);
+        log_debug(LD_REND, "Adding client name '%s'", client_name);
+      }
+      SMARTLIST_FOREACH_END(client_name);
+      SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
+      smartlist_free(clients);
+      /* Ensure maximum number of clients. */
+      if ((service->auth_type == REND_BASIC_AUTH &&
+            smartlist_len(service->clients) > 512) ||
+          (service->auth_type == REND_STEALTH_AUTH &&
+            smartlist_len(service->clients) > 16)) {
+        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
+                            "client authorization entries, but only a "
+                            "maximum of %d entries is allowed for "
+                            "authorization type '%s'.",
+                 smartlist_len(service->clients),
+                 service->auth_type == REND_BASIC_AUTH ? 512 : 16,
+                 service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
+        goto err;
+      }
+      continue;
+    }
+  }
+  /* Validate the service just parsed. */
+  if (rend_validate_service(rend_service_staging_list, service) < 0) {
+    /* Service is in the staging list so don't try to free it. */
+    goto err;
+  }
+
+  /* Add it to the temporary list which we will use to prune our current
+   * list if any after configuring all services. */
+  if (rend_add_service(rend_service_staging_list, service) < 0) {
+    /* The object has been freed on error already. */
+    service = NULL;
+    goto err;
+  }
+
+  return 0;
+ err:
+  rend_service_free(service);
+  return -1;
+}
+
+/** Add the ephemeral service <b>pk</b>/<b>ports</b> if possible, using
+ * client authorization <b>auth_type</b> and an optional list of
+ * rend_authorized_client_t in <b>auth_clients</b>, with
+ * <b>max_streams_per_circuit</b> streams allowed per rendezvous circuit,
+ * and circuit closure on max streams being exceeded set by
+ * <b>max_streams_close_circuit</b>.
+ *
+ * Ownership of pk, ports, and auth_clients is passed to this routine.
+ * Regardless of success/failure, callers should not touch these values
+ * after calling this routine, and may assume that correct cleanup has
+ * been done on failure.
+ *
+ * Return an appropriate hs_service_add_ephemeral_status_t.
+ */
+hs_service_add_ephemeral_status_t
+rend_service_add_ephemeral(crypto_pk_t *pk,
+                           smartlist_t *ports,
+                           int max_streams_per_circuit,
+                           int max_streams_close_circuit,
+                           rend_auth_type_t auth_type,
+                           smartlist_t *auth_clients,
+                           char **service_id_out)
+{
+  *service_id_out = NULL;
+  /* Allocate the service structure, and initialize the key, and key derived
+   * parameters.
+   */
+  rend_service_t *s = tor_malloc_zero(sizeof(rend_service_t));
+  s->directory = NULL; /* This indicates the service is ephemeral. */
+  s->private_key = pk;
+  s->auth_type = auth_type;
+  s->clients = auth_clients;
+  s->ports = ports;
+  s->intro_period_started = time(NULL);
+  s->n_intro_points_wanted = NUM_INTRO_POINTS_DEFAULT;
+  s->max_streams_per_circuit = max_streams_per_circuit;
+  s->max_streams_close_circuit = max_streams_close_circuit;
+  if (rend_service_derive_key_digests(s) < 0) {
+    rend_service_free(s);
+    return RSAE_BADPRIVKEY;
+  }
+
+  if (!s->ports || smartlist_len(s->ports) == 0) {
+    log_warn(LD_CONFIG, "At least one VIRTPORT/TARGET must be specified.");
+    rend_service_free(s);
+    return RSAE_BADVIRTPORT;
+  }
+  if (s->auth_type != REND_NO_AUTH &&
+      (!s->clients || smartlist_len(s->clients) == 0)) {
+    log_warn(LD_CONFIG, "At least one authorized client must be specified.");
+    rend_service_free(s);
+    return RSAE_BADAUTH;
+  }
+
+  /* Enforcing pk/id uniqueness should be done by rend_service_load_keys(), but
+   * it's not, see #14828.
+   */
+  if (rend_service_get_by_pk_digest(s->pk_digest)) {
+    log_warn(LD_CONFIG, "Onion Service private key collides with an "
+             "existing service.");
+    rend_service_free(s);
+    return RSAE_ADDREXISTS;
+  }
+  if (rend_service_get_by_service_id(s->service_id)) {
+    log_warn(LD_CONFIG, "Onion Service id collides with an existing service.");
+    rend_service_free(s);
+    return RSAE_ADDREXISTS;
+  }
+
+  /* Initialize the service. */
+  if (rend_add_service(NULL, s)) {
+    return RSAE_INTERNAL;
+  }
+  *service_id_out = tor_strdup(s->service_id);
+
+  log_debug(LD_CONFIG, "Added ephemeral Onion Service: %s", s->service_id);
+  return RSAE_OKAY;
+}
+
+/** Remove the ephemeral service <b>service_id</b> if possible.  Returns 0 on
+ * success, and -1 on failure.
+ */
+int
+rend_service_del_ephemeral(const char *service_id)
+{
+  rend_service_t *s;
+  if (!rend_valid_v2_service_id(service_id)) {
+    log_warn(LD_CONFIG, "Requested malformed Onion Service id for removal.");
+    return -1;
+  }
+  if ((s = rend_service_get_by_service_id(service_id)) == NULL) {
+    log_warn(LD_CONFIG, "Requested non-existent Onion Service id for "
+             "removal.");
+    return -1;
+  }
+  if (!rend_service_is_ephemeral(s)) {
+    log_warn(LD_CONFIG, "Requested non-ephemeral Onion Service for removal.");
+    return -1;
+  }
+
+  /* Kill the intro point circuit for the Onion Service, and remove it from
+   * the list.  Closing existing connections is the application's problem.
+   *
+   * XXX: As with the comment in rend_config_services(), a nice abstraction
+   * would be ideal here, but for now just duplicate the code.
+   */
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!circ->marked_for_close &&
+        (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+         circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
+      origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+      if (oc->rend_data == NULL ||
+          !rend_circuit_pk_digest_eq(oc, (uint8_t *) s->pk_digest)) {
+        continue;
+      }
+      log_debug(LD_REND, "Closing intro point %s for service %s.",
+                safe_str_client(extend_info_describe(
+                                          oc->build_state->chosen_exit)),
+                rend_data_get_address(oc->rend_data));
+      circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
+    }
+  } SMARTLIST_FOREACH_END(circ);
+  smartlist_remove(rend_service_list, s);
+  /* Notify that we just removed a service from our global list. */
+  hs_service_map_has_changed();
+  rend_service_free(s);
+
+  log_debug(LD_CONFIG, "Removed ephemeral Onion Service: %s", service_id);
+
+  return 0;
+}
+
+/* There can be 1 second's delay due to second_elapsed_callback, and perhaps
+ * another few seconds due to blocking calls. */
+#define INTRO_CIRC_RETRY_PERIOD_SLOP 10
+
+/** Log information about the intro point creation rate and current intro
+ * points for service, upgrading the log level from min_severity to warn if
+ * we have stopped launching new intro point circuits. */
+static void
+rend_log_intro_limit(const rend_service_t *service, int min_severity)
+{
+  int exceeded_limit = (service->n_intro_circuits_launched >=
+                        rend_max_intro_circs_per_period(
+                                              service->n_intro_points_wanted));
+  int severity = min_severity;
+  /* We stopped creating circuits */
+  if (exceeded_limit) {
+    severity = LOG_WARN;
+  }
+  time_t intro_period_elapsed = time(NULL) - service->intro_period_started;
+  tor_assert_nonfatal(intro_period_elapsed >= 0);
+  {
+    char *msg;
+    static ratelim_t rlimit = RATELIM_INIT(INTRO_CIRC_RETRY_PERIOD);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_fn(severity, LD_REND,
+             "Hidden service %s %s %d intro points in the last %d seconds. "
+             "Intro circuit launches are limited to %d per %d seconds.%s",
+             service->service_id,
+             exceeded_limit ? "exceeded launch limit with" : "launched",
+             service->n_intro_circuits_launched,
+             (int)intro_period_elapsed,
+             rend_max_intro_circs_per_period(service->n_intro_points_wanted),
+             INTRO_CIRC_RETRY_PERIOD, msg);
+      rend_service_dump_stats(severity);
+      tor_free(msg);
+    }
+  }
+}
+
+/** Replace the old value of <b>service</b>-\>desc with one that reflects
+ * the other fields in service.
+ */
+static void
+rend_service_update_descriptor(rend_service_t *service)
+{
+  rend_service_descriptor_t *d;
+  int i;
+
+  rend_service_descriptor_free(service->desc);
+  service->desc = NULL;
+
+  d = service->desc = tor_malloc_zero(sizeof(rend_service_descriptor_t));
+  d->pk = crypto_pk_dup_key(service->private_key);
+  d->timestamp = time(NULL);
+  d->timestamp -= d->timestamp % 3600; /* Round down to nearest hour */
+  d->intro_nodes = smartlist_new();
+  /* Support intro protocols 2 and 3. */
+  d->protocols = (1 << 2) + (1 << 3);
+
+  for (i = 0; i < smartlist_len(service->intro_nodes); ++i) {
+    rend_intro_point_t *intro_svc = smartlist_get(service->intro_nodes, i);
+    rend_intro_point_t *intro_desc;
+
+    /* This intro point won't be listed in the descriptor... */
+    intro_svc->listed_in_last_desc = 0;
+
+    /* circuit_established is set in rend_service_intro_established(), and
+     * checked every second in rend_consider_services_intro_points(), so it's
+     * safe to use it here */
+    if (!intro_svc->circuit_established) {
+      continue;
+    }
+
+    /* ...unless this intro point is listed in the descriptor. */
+    intro_svc->listed_in_last_desc = 1;
+
+    /* We have an entirely established intro circuit.  Publish it in
+     * our descriptor. */
+    intro_desc = tor_malloc_zero(sizeof(rend_intro_point_t));
+    intro_desc->extend_info = extend_info_dup(intro_svc->extend_info);
+    if (intro_svc->intro_key)
+      intro_desc->intro_key = crypto_pk_dup_key(intro_svc->intro_key);
+    smartlist_add(d->intro_nodes, intro_desc);
+
+    if (intro_svc->time_published == -1) {
+      /* We are publishing this intro point in a descriptor for the
+       * first time -- note the current time in the service's copy of
+       * the intro point. */
+      intro_svc->time_published = time(NULL);
+    }
+  }
+
+  /* Check that we have the right number of intro points */
+  unsigned int have_intro = (unsigned int)smartlist_len(d->intro_nodes);
+  if (have_intro != service->n_intro_points_wanted) {
+    int severity;
+    /* Getting less than we wanted or more than we're allowed is serious */
+    if (have_intro < service->n_intro_points_wanted ||
+        have_intro > NUM_INTRO_POINTS_MAX) {
+      severity = LOG_WARN;
+    } else {
+      /* Getting more than we wanted is weird, but less of a problem */
+      severity = LOG_NOTICE;
+    }
+    log_fn(severity, LD_REND, "Hidden service %s wanted %d intro points, but "
+           "descriptor was updated with %d instead.",
+           service->service_id,
+           service->n_intro_points_wanted, have_intro);
+    /* Now log an informative message about how we might have got here. */
+    rend_log_intro_limit(service, severity);
+  }
+}
+
+/* Allocate and return a string containing the path to file_name in
+ * service->directory. Asserts that service has a directory.
+ * This function will never return NULL.
+ * The caller must free this path. */
+static char *
+rend_service_path(const rend_service_t *service, const char *file_name)
+{
+  tor_assert(service->directory);
+  return hs_path_from_filename(service->directory, file_name);
+}
+
+/* Allocate and return a string containing the path to the single onion
+ * service poison file in service->directory. Asserts that service has a
+ * directory.
+ * The caller must free this path. */
+STATIC char *
+rend_service_sos_poison_path(const rend_service_t *service)
+{
+  return rend_service_path(service, sos_poison_fname);
+}
+
+/** Return True if hidden services <b>service</b> has been poisoned by single
+ * onion mode. */
+static int
+service_is_single_onion_poisoned(const rend_service_t *service)
+{
+  char *poison_fname = NULL;
+  file_status_t fstatus;
+
+  /* Passing a NULL service is a bug */
+  if (BUG(!service)) {
+    return 0;
+  }
+
+  if (rend_service_is_ephemeral(service)) {
+    return 0;
+  }
+
+  poison_fname = rend_service_sos_poison_path(service);
+
+  fstatus = file_status(poison_fname);
+  tor_free(poison_fname);
+
+  /* If this fname is occupied, the hidden service has been poisoned.
+   * fstatus can be FN_ERROR if the service directory does not exist, in that
+   * case, there is obviously no private key. */
+  if (fstatus == FN_FILE || fstatus == FN_EMPTY) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/* Return 1 if the private key file for service exists and has a non-zero size,
+ * and 0 otherwise. */
+static int
+rend_service_private_key_exists(const rend_service_t *service)
+{
+  char *private_key_path = rend_service_path(service, private_key_fname);
+  const file_status_t private_key_status = file_status(private_key_path);
+  tor_free(private_key_path);
+  /* Only non-empty regular private key files could have been used before.
+   * fstatus can be FN_ERROR if the service directory does not exist, in that
+   * case, there is obviously no private key. */
+  return private_key_status == FN_FILE;
+}
+
+/** Check the single onion service poison state of the directory for s:
+ * - If the service is poisoned, and we are in Single Onion Mode,
+ *   return 0,
+ * - If the service is not poisoned, and we are not in Single Onion Mode,
+ *   return 0,
+ * - Otherwise, the poison state is invalid: the service was created in one
+ *   mode, and is being used in the other, return -1.
+ * Hidden service directories without keys are always considered consistent.
+ * They will be poisoned after their directory is created (if needed). */
+STATIC int
+rend_service_verify_single_onion_poison(const rend_service_t* s,
+                                        const or_options_t* options)
+{
+  /* Passing a NULL service is a bug */
+  if (BUG(!s)) {
+    return -1;
+  }
+
+  /* Ephemeral services are checked at ADD_ONION time */
+  if (BUG(rend_service_is_ephemeral(s))) {
+    return -1;
+  }
+
+  /* Service is expected to have a directory */
+  if (BUG(!s->directory)) {
+    return -1;
+  }
+
+  /* Services without keys are always ok - their keys will only ever be used
+   * in the current mode */
+  if (!rend_service_private_key_exists(s)) {
+    return 0;
+  }
+
+  /* The key has been used before in a different mode */
+  if (service_is_single_onion_poisoned(s) !=
+      rend_service_non_anonymous_mode_enabled(options)) {
+    return -1;
+  }
+
+  /* The key exists and is consistent with the current mode */
+  return 0;
+}
+
+/*** Helper for rend_service_poison_new_single_onion_dir(). Add a file to
+ * the hidden service directory for s that marks it as a single onion service.
+ * Tor must be in single onion mode before calling this function, and the
+ * service directory must already have been created.
+ * Returns 0 when a directory is successfully poisoned, or if it is already
+ * poisoned. Returns -1 on a failure to read the directory or write the poison
+ * file, or if there is an existing private key file in the directory. (The
+ * service should have been poisoned when the key was created.) */
+static int
+poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
+                                                const or_options_t* options)
+{
+  /* Passing a NULL service is a bug */
+  if (BUG(!service)) {
+    return -1;
+  }
+
+  /* We must only poison directories if we're in Single Onion mode */
+  tor_assert(rend_service_non_anonymous_mode_enabled(options));
+
+  int fd;
+  int retval = -1;
+  char *poison_fname = NULL;
+
+  if (rend_service_is_ephemeral(service)) {
+    log_info(LD_REND, "Ephemeral HS started in non-anonymous mode.");
+    return 0;
+  }
+
+  /* Make sure we're only poisoning new hidden service directories */
+  if (rend_service_private_key_exists(service)) {
+    log_warn(LD_BUG, "Tried to single onion poison a service directory after "
+             "the private key was created.");
+    return -1;
+  }
+
+  /* Make sure the directory was created before calling this function. */
+  if (BUG(hs_check_service_private_dir(options->User, service->directory,
+                                       service->dir_group_readable, 0) < 0))
+    return -1;
+
+  poison_fname = rend_service_sos_poison_path(service);
+
+  switch (file_status(poison_fname)) {
+  case FN_DIR:
+  case FN_ERROR:
+    log_warn(LD_FS, "Can't read single onion poison file \"%s\"",
+             poison_fname);
+    goto done;
+  case FN_FILE: /* single onion poison file already exists. NOP. */
+  case FN_EMPTY: /* single onion poison file already exists. NOP. */
+    log_debug(LD_FS, "Tried to re-poison a single onion poisoned file \"%s\"",
+              poison_fname);
+    break;
+  case FN_NOENT:
+    fd = tor_open_cloexec(poison_fname, O_RDWR|O_CREAT|O_TRUNC, 0600);
+    if (fd < 0) {
+      log_warn(LD_FS, "Could not create single onion poison file %s",
+               poison_fname);
+      goto done;
+    }
+    close(fd);
+    break;
+  default:
+    tor_assert(0);
+  }
+
+  retval = 0;
+
+ done:
+  tor_free(poison_fname);
+
+  return retval;
+}
+
+/** We just got launched in Single Onion Mode. That's a non-anonymous mode for
+ * hidden services. If s is new, we should mark its hidden service
+ * directory appropriately so that it is never launched as a location-private
+ * hidden service. (New directories don't have private key files.)
+ * Return 0 on success, -1 on fail. */
+STATIC int
+rend_service_poison_new_single_onion_dir(const rend_service_t *s,
+                                         const or_options_t* options)
+{
+  /* Passing a NULL service is a bug */
+  if (BUG(!s)) {
+    return -1;
+  }
+
+  /* We must only poison directories if we're in Single Onion mode */
+  tor_assert(rend_service_non_anonymous_mode_enabled(options));
+
+  /* Ephemeral services aren't allowed in non-anonymous mode */
+  if (BUG(rend_service_is_ephemeral(s))) {
+    return -1;
+  }
+
+  /* Service is expected to have a directory */
+  if (BUG(!s->directory)) {
+    return -1;
+  }
+
+  if (!rend_service_private_key_exists(s)) {
+    if (poison_new_single_onion_hidden_service_dir_impl(s, options)
+        < 0) {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/** Load and/or generate private keys for all hidden services, possibly
+ * including keys for client authorization.
+ * If a <b>service_list</b> is provided, treat it as the list of hidden
+ * services (used in unittests). Otherwise, require that rend_service_list is
+ * not NULL.
+ * Return 0 on success, -1 on failure. */
+int
+rend_service_load_all_keys(const smartlist_t *service_list)
+{
+  /* Use service_list for unit tests */
+  const smartlist_t *s_list = rend_get_service_list(service_list);
+  if (BUG(!s_list)) {
+    return -1;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(s_list, rend_service_t *, s) {
+    if (s->private_key)
+      continue;
+    log_info(LD_REND, "Loading hidden-service keys from %s",
+             rend_service_escaped_dir(s));
+
+    if (rend_service_load_keys(s) < 0)
+      return -1;
+  } SMARTLIST_FOREACH_END(s);
+
+  return 0;
+}
+
+/** Add to <b>lst</b> every filename used by <b>s</b>. */
+static void
+rend_service_add_filenames_to_list(smartlist_t *lst, const rend_service_t *s)
+{
+  tor_assert(lst);
+  tor_assert(s);
+  tor_assert(s->directory);
+  smartlist_add(lst, rend_service_path(s, private_key_fname));
+  smartlist_add(lst, rend_service_path(s, hostname_fname));
+  smartlist_add(lst, rend_service_path(s, client_keys_fname));
+  smartlist_add(lst, rend_service_sos_poison_path(s));
+}
+
+/** Add to <b>open_lst</b> every filename used by a configured hidden service,
+ * and to <b>stat_lst</b> every directory used by a configured hidden
+ * service */
+void
+rend_services_add_filenames_to_lists(smartlist_t *open_lst,
+                                     smartlist_t *stat_lst)
+{
+  if (!rend_service_list)
+    return;
+  SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, s) {
+    if (!rend_service_is_ephemeral(s)) {
+      rend_service_add_filenames_to_list(open_lst, s);
+      smartlist_add_strdup(stat_lst, s->directory);
+    }
+  } SMARTLIST_FOREACH_END(s);
+}
+
+/** Derive all rend_service_t internal material based on the service's key.
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+rend_service_derive_key_digests(struct rend_service_t *s)
+{
+  if (rend_get_service_id(s->private_key, s->service_id)<0) {
+    log_warn(LD_BUG, "Internal error: couldn't encode service ID.");
+    return -1;
+  }
+  if (crypto_pk_get_digest(s->private_key, s->pk_digest)<0) {
+    log_warn(LD_BUG, "Couldn't compute hash of public key.");
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Make sure that the directory for <b>s</b> is private, using the config in
+ * <b>options</b>.
+ * If <b>create</b> is true:
+ *  - if the directory exists, change permissions if needed,
+ *  - if the directory does not exist, create it with the correct permissions.
+ * If <b>create</b> is false:
+ *  - if the directory exists, check permissions,
+ *  - if the directory does not exist, check if we think we can create it.
+ * Return 0 on success, -1 on failure. */
+static int
+rend_service_check_private_dir(const or_options_t *options,
+                               const rend_service_t *s,
+                               int create)
+{
+  /* Passing a NULL service is a bug */
+  if (BUG(!s)) {
+    return -1;
+  }
+
+  /* Check/create directory */
+  if (hs_check_service_private_dir(options->User, s->directory,
+                                   s->dir_group_readable, create) < 0) {
+    return -1;
+  }
+
+  /* Check if the hidden service key exists, and was created in a different
+   * single onion service mode, and refuse to launch if it has.
+   * This is safe to call even when create is false, as it ignores missing
+   * keys and directories: they are always valid.
+   */
+  if (rend_service_verify_single_onion_poison(s, options) < 0) {
+    /* We can't use s->service_id here, as the key may not have been loaded */
+    log_warn(LD_GENERAL, "We are configured with "
+             "HiddenServiceNonAnonymousMode %d, but the hidden "
+             "service key in directory %s was created in %s mode. "
+             "This is not allowed.",
+             rend_service_non_anonymous_mode_enabled(options) ? 1 : 0,
+             rend_service_escaped_dir(s),
+             rend_service_non_anonymous_mode_enabled(options) ?
+             "an anonymous" : "a non-anonymous"
+             );
+    return -1;
+  }
+
+  /* Poison new single onion directories immediately after they are created,
+   * so that we never accidentally launch non-anonymous hidden services
+   * thinking they are anonymous. Any keys created later will end up with the
+   * correct poisoning state.
+   */
+  if (create && rend_service_non_anonymous_mode_enabled(options)) {
+    static int logged_warning = 0;
+
+    if (rend_service_poison_new_single_onion_dir(s, options) < 0) {
+      log_warn(LD_GENERAL,"Failed to mark new hidden services as non-anonymous"
+               ".");
+      return -1;
+    }
+
+    if (!logged_warning) {
+      /* The keys for these services are linked to the server IP address */
+      log_notice(LD_REND, "The configured onion service directories have been "
+                 "used in single onion mode. They can not be used for "
+                 "anonymous hidden services.");
+      logged_warning = 1;
+    }
+  }
+
+  return 0;
+}
+
+/** Load and/or generate private keys for the hidden service <b>s</b>,
+ * possibly including keys for client authorization.  Return 0 on success, -1
+ * on failure. */
+static int
+rend_service_load_keys(rend_service_t *s)
+{
+  char *fname = NULL;
+  char buf[128];
+
+  /* Create the directory if needed which will also poison it in case of
+   * single onion service. */
+  if (rend_service_check_private_dir(get_options(), s, 1) < 0)
+    goto err;
+
+  /* Load key */
+  fname = rend_service_path(s, private_key_fname);
+  s->private_key = init_key_from_file(fname, 1, LOG_ERR, 0);
+
+  if (!s->private_key)
+    goto err;
+
+  if (rend_service_derive_key_digests(s) < 0)
+    goto err;
+
+  tor_free(fname);
+  /* Create service file */
+  fname = rend_service_path(s, hostname_fname);
+
+  tor_snprintf(buf, sizeof(buf),"%s.onion\n", s->service_id);
+  if (write_str_to_file(fname,buf,0)<0) {
+    log_warn(LD_CONFIG, "Could not write onion address to hostname file.");
+    goto err;
+  }
+#ifndef _WIN32
+  if (s->dir_group_readable) {
+    /* Also verify hostname file created with group read. */
+    if (chmod(fname, 0640))
+      log_warn(LD_FS,"Unable to make hidden hostname file %s group-readable.",
+               fname);
+  }
+#endif /* !defined(_WIN32) */
+
+  /* If client authorization is configured, load or generate keys. */
+  if (s->auth_type != REND_NO_AUTH) {
+    if (rend_service_load_auth_keys(s, fname) < 0) {
+      goto err;
+    }
+  }
+
+  int r = 0;
+  goto done;
+ err:
+  r = -1;
+ done:
+  memwipe(buf, 0, sizeof(buf));
+  tor_free(fname);
+  return r;
+}
+
+/** Load and/or generate client authorization keys for the hidden service
+ * <b>s</b>, which stores its hostname in <b>hfname</b>.  Return 0 on success,
+ * -1 on failure. */
+static int
+rend_service_load_auth_keys(rend_service_t *s, const char *hfname)
+{
+  int r = 0;
+  char *cfname = NULL;
+  char *client_keys_str = NULL;
+  strmap_t *parsed_clients = strmap_new();
+  FILE *cfile, *hfile;
+  open_file_t *open_cfile = NULL, *open_hfile = NULL;
+  char desc_cook_out[3*REND_DESC_COOKIE_LEN_BASE64+1];
+  char service_id[16+1];
+  char buf[1500];
+
+  /* Load client keys and descriptor cookies, if available. */
+  cfname = rend_service_path(s, client_keys_fname);
+  client_keys_str = read_file_to_str(cfname, RFTS_IGNORE_MISSING, NULL);
+  if (client_keys_str) {
+    if (rend_parse_client_keys(parsed_clients, client_keys_str) < 0) {
+      log_warn(LD_CONFIG, "Previously stored client_keys file could not "
+               "be parsed.");
+      goto err;
+    } else {
+      log_info(LD_CONFIG, "Parsed %d previously stored client entries.",
+               strmap_size(parsed_clients));
+    }
+  }
+
+  /* Prepare client_keys and hostname files. */
+  if (!(cfile = start_writing_to_stdio_file(cfname,
+                                            OPEN_FLAGS_REPLACE | O_TEXT,
+                                            0600, &open_cfile))) {
+    log_warn(LD_CONFIG, "Could not open client_keys file %s",
+             escaped(cfname));
+    goto err;
+  }
+
+  if (!(hfile = start_writing_to_stdio_file(hfname,
+                                            OPEN_FLAGS_REPLACE | O_TEXT,
+                                            0600, &open_hfile))) {
+    log_warn(LD_CONFIG, "Could not open hostname file %s", escaped(hfname));
+    goto err;
+  }
+
+  /* Either use loaded keys for configured clients or generate new
+   * ones if a client is new. */
+  SMARTLIST_FOREACH_BEGIN(s->clients, rend_authorized_client_t *, client) {
+    rend_authorized_client_t *parsed =
+      strmap_get(parsed_clients, client->client_name);
+    int written;
+    size_t len;
+    /* Copy descriptor cookie from parsed entry or create new one. */
+    if (parsed) {
+      memcpy(client->descriptor_cookie, parsed->descriptor_cookie,
+             REND_DESC_COOKIE_LEN);
+    } else {
+      crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
+    }
+    /* For compatibility with older tor clients, this does not
+     * truncate the padding characters, unlike rend_auth_encode_cookie.  */
+    if (base64_encode(desc_cook_out, 3*REND_DESC_COOKIE_LEN_BASE64+1,
+                      (char *) client->descriptor_cookie,
+                      REND_DESC_COOKIE_LEN, 0) < 0) {
+      log_warn(LD_BUG, "Could not base64-encode descriptor cookie.");
+      goto err;
+    }
+    /* Copy client key from parsed entry or create new one if required. */
+    if (parsed && parsed->client_key) {
+      client->client_key = crypto_pk_dup_key(parsed->client_key);
+    } else if (s->auth_type == REND_STEALTH_AUTH) {
+      /* Create private key for client. */
+      crypto_pk_t *prkey = NULL;
+      if (!(prkey = crypto_pk_new())) {
+        log_warn(LD_BUG,"Error constructing client key");
+        goto err;
+      }
+      if (crypto_pk_generate_key(prkey)) {
+        log_warn(LD_BUG,"Error generating client key");
+        crypto_pk_free(prkey);
+        goto err;
+      }
+      if (crypto_pk_check_key(prkey) <= 0) {
+        log_warn(LD_BUG,"Generated client key seems invalid");
+        crypto_pk_free(prkey);
+        goto err;
+      }
+      client->client_key = prkey;
+    }
+    /* Add entry to client_keys file. */
+    written = tor_snprintf(buf, sizeof(buf),
+                           "client-name %s\ndescriptor-cookie %s\n",
+                           client->client_name, desc_cook_out);
+    if (written < 0) {
+      log_warn(LD_BUG, "Could not write client entry.");
+      goto err;
+    }
+    if (client->client_key) {
+      char *client_key_out = NULL;
+      if (crypto_pk_write_private_key_to_string(client->client_key,
+                                                &client_key_out, &len) != 0) {
+        log_warn(LD_BUG, "Internal error: "
+                 "crypto_pk_write_private_key_to_string() failed.");
+        goto err;
+      }
+      if (rend_get_service_id(client->client_key, service_id)<0) {
+        log_warn(LD_BUG, "Internal error: couldn't encode service ID.");
+        /*
+         * len is string length, not buffer length, but last byte is NUL
+         * anyway.
+         */
+        memwipe(client_key_out, 0, len);
+        tor_free(client_key_out);
+        goto err;
+      }
+      written = tor_snprintf(buf + written, sizeof(buf) - written,
+                             "client-key\n%s", client_key_out);
+      memwipe(client_key_out, 0, len);
+      tor_free(client_key_out);
+      if (written < 0) {
+        log_warn(LD_BUG, "Could not write client entry.");
+        goto err;
+      }
+    } else {
+      strlcpy(service_id, s->service_id, sizeof(service_id));
+    }
+
+    if (fputs(buf, cfile) < 0) {
+      log_warn(LD_FS, "Could not append client entry to file: %s",
+               strerror(errno));
+      goto err;
+    }
+
+    /* Add line to hostname file. This is not the same encoding as in
+     * client_keys. */
+    char *encoded_cookie = rend_auth_encode_cookie(client->descriptor_cookie,
+                                                   s->auth_type);
+    if (!encoded_cookie) {
+      log_warn(LD_BUG, "Could not base64-encode descriptor cookie.");
+      goto err;
+    }
+    tor_snprintf(buf, sizeof(buf), "%s.onion %s # client: %s\n",
+                 service_id, encoded_cookie, client->client_name);
+    memwipe(encoded_cookie, 0, strlen(encoded_cookie));
+    tor_free(encoded_cookie);
+
+    if (fputs(buf, hfile)<0) {
+      log_warn(LD_FS, "Could not append host entry to file: %s",
+               strerror(errno));
+      goto err;
+    }
+  } SMARTLIST_FOREACH_END(client);
+
+  finish_writing_to_file(open_cfile);
+  finish_writing_to_file(open_hfile);
+
+  goto done;
+ err:
+  r = -1;
+  if (open_cfile)
+    abort_writing_to_file(open_cfile);
+  if (open_hfile)
+    abort_writing_to_file(open_hfile);
+ done:
+  if (client_keys_str) {
+    memwipe(client_keys_str, 0, strlen(client_keys_str));
+    tor_free(client_keys_str);
+  }
+  strmap_free(parsed_clients, rend_authorized_client_free_void);
+
+  if (cfname) {
+    memwipe(cfname, 0, strlen(cfname));
+    tor_free(cfname);
+  }
+
+  /* Clear stack buffers that held key-derived material. */
+  memwipe(buf, 0, sizeof(buf));
+  memwipe(desc_cook_out, 0, sizeof(desc_cook_out));
+  memwipe(service_id, 0, sizeof(service_id));
+
+  return r;
+}
+
+/** Return the service whose public key has a digest of <b>digest</b>, or
+ * NULL if no such service exists.
+ */
+static rend_service_t *
+rend_service_get_by_pk_digest(const char* digest)
+{
+  SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s,
+                    if (tor_memeq(s->pk_digest,digest,DIGEST_LEN))
+                        return s);
+  return NULL;
+}
+
+/** Return the service whose service id is <b>id</b>, or NULL if no such
+ * service exists.
+ */
+static struct rend_service_t *
+rend_service_get_by_service_id(const char *id)
+{
+  tor_assert(strlen(id) == REND_SERVICE_ID_LEN_BASE32);
+  SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, {
+    if (tor_memeq(s->service_id, id, REND_SERVICE_ID_LEN_BASE32))
+      return s;
+  });
+  return NULL;
+}
+
+/** Check client authorization of a given <b>descriptor_cookie</b> of
+ * length <b>cookie_len</b> for <b>service</b>. Return 1 for success
+ * and 0 for failure. */
+static int
+rend_check_authorization(rend_service_t *service,
+                         const char *descriptor_cookie,
+                         size_t cookie_len)
+{
+  rend_authorized_client_t *auth_client = NULL;
+  tor_assert(service);
+  tor_assert(descriptor_cookie);
+  if (!service->clients) {
+    log_warn(LD_BUG, "Can't check authorization for a service that has no "
+                     "authorized clients configured.");
+    return 0;
+  }
+
+  if (cookie_len != REND_DESC_COOKIE_LEN) {
+    log_info(LD_REND, "Descriptor cookie is %lu bytes, but we expected "
+                      "%lu bytes. Dropping cell.",
+             (unsigned long)cookie_len, (unsigned long)REND_DESC_COOKIE_LEN);
+    return 0;
+  }
+
+  /* Look up client authorization by descriptor cookie. */
+  SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, client, {
+    if (tor_memeq(client->descriptor_cookie, descriptor_cookie,
+                REND_DESC_COOKIE_LEN)) {
+      auth_client = client;
+      break;
+    }
+  });
+  if (!auth_client) {
+    char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64];
+    base64_encode(descriptor_cookie_base64, sizeof(descriptor_cookie_base64),
+                  descriptor_cookie, REND_DESC_COOKIE_LEN, 0);
+    log_info(LD_REND, "No authorization found for descriptor cookie '%s'! "
+                      "Dropping cell!",
+             descriptor_cookie_base64);
+    return 0;
+  }
+
+  /* Allow the request. */
+  log_info(LD_REND, "Client %s authorized for service %s.",
+           auth_client->client_name, service->service_id);
+  return 1;
+}
+
+/* Can this service make a direct connection to ei?
+ * It must be a single onion service, and the firewall rules must allow ei. */
+static int
+rend_service_use_direct_connection(const or_options_t* options,
+                                   const extend_info_t* ei)
+{
+  /* We'll connect directly all reachable addresses, whether preferred or not.
+   * The prefer_ipv6 argument to fascist_firewall_allows_address_addr is
+   * ignored, because pref_only is 0. */
+  return (rend_service_allow_non_anonymous_connection(options) &&
+          fascist_firewall_allows_address_addr(&ei->addr, ei->port,
+                                               FIREWALL_OR_CONNECTION, 0, 0));
+}
+
+/* Like rend_service_use_direct_connection, but to a node. */
+static int
+rend_service_use_direct_connection_node(const or_options_t* options,
+                                        const node_t* node)
+{
+  /* We'll connect directly all reachable addresses, whether preferred or not.
+   */
+  return (rend_service_allow_non_anonymous_connection(options) &&
+          fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0));
+}
+
+/******
+ * Handle cells
+ ******/
+
+/** Respond to an INTRODUCE2 cell by launching a circuit to the chosen
+ * rendezvous point.
+ */
+int
+rend_service_receive_introduction(origin_circuit_t *circuit,
+                                  const uint8_t *request,
+                                  size_t request_len)
+{
+  /* Global status stuff */
+  int status = 0, result;
+  const or_options_t *options = get_options();
+  char *err_msg = NULL;
+  int err_msg_severity = LOG_WARN;
+  const char *stage_descr = NULL, *rend_pk_digest;
+  int reason = END_CIRC_REASON_TORPROTOCOL;
+  /* Service/circuit/key stuff we can learn before parsing */
+  char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  rend_service_t *service = NULL;
+  rend_intro_point_t *intro_point = NULL;
+  crypto_pk_t *intro_key = NULL;
+  /* Parsed cell */
+  rend_intro_cell_t *parsed_req = NULL;
+  /* Rendezvous point */
+  extend_info_t *rp = NULL;
+  /* XXX not handled yet */
+  char buf[RELAY_PAYLOAD_SIZE];
+  char keys[DIGEST_LEN+CPATH_KEY_MATERIAL_LEN]; /* Holds KH, Df, Db, Kf, Kb */
+  int i;
+  crypto_dh_t *dh = NULL;
+  origin_circuit_t *launched = NULL;
+  crypt_path_t *cpath = NULL;
+  char hexcookie[9];
+  int circ_needs_uptime;
+  time_t now = time(NULL);
+  time_t elapsed;
+  int replay;
+  ssize_t keylen;
+
+  /* Do some initial validation and logging before we parse the cell */
+  if (circuit->base_.purpose != CIRCUIT_PURPOSE_S_INTRO) {
+    log_warn(LD_PROTOCOL,
+             "Got an INTRODUCE2 over a non-introduction circuit %u.",
+             (unsigned) circuit->base_.n_circ_id);
+    goto err;
+  }
+
+  assert_circ_anonymity_ok(circuit, options);
+  tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only one supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
+
+  /* We'll use this in a bazillion log messages */
+  base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                rend_pk_digest, REND_SERVICE_ID_LEN);
+
+  /* look up service depending on circuit. */
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
+  if (!service) {
+    log_warn(LD_BUG,
+             "Internal error: Got an INTRODUCE2 cell on an intro "
+             "circ for an unrecognized service %s.",
+             escaped(serviceid));
+    goto err;
+  }
+
+  intro_point = find_intro_point(circuit);
+  if (intro_point == NULL) {
+    intro_point = find_expiring_intro_point(service, circuit);
+    if (intro_point == NULL) {
+      log_warn(LD_BUG,
+               "Internal error: Got an INTRODUCE2 cell on an "
+               "intro circ (for service %s) with no corresponding "
+               "rend_intro_point_t.",
+               escaped(serviceid));
+      goto err;
+    }
+  }
+
+  log_info(LD_REND, "Received INTRODUCE2 cell for service %s on circ %u.",
+           escaped(serviceid), (unsigned)circuit->base_.n_circ_id);
+
+  /* use intro key instead of service key. */
+  intro_key = circuit->intro_key;
+
+  tor_free(err_msg);
+  stage_descr = NULL;
+
+  stage_descr = "early parsing";
+  /* Early parsing pass (get pk, ciphertext); type 2 is INTRODUCE2 */
+  parsed_req =
+    rend_service_begin_parse_intro(request, request_len, 2, &err_msg);
+  if (!parsed_req) {
+    goto log_error;
+  } else if (err_msg) {
+    log_info(LD_REND, "%s on circ %u.", err_msg,
+             (unsigned)circuit->base_.n_circ_id);
+    tor_free(err_msg);
+  }
+
+  /* make sure service replay caches are present */
+  if (!service->accepted_intro_dh_parts) {
+    service->accepted_intro_dh_parts =
+      replaycache_new(REND_REPLAY_TIME_INTERVAL,
+                      REND_REPLAY_TIME_INTERVAL);
+  }
+
+  if (!intro_point->accepted_intro_rsa_parts) {
+    intro_point->accepted_intro_rsa_parts = replaycache_new(0, 0);
+  }
+
+  /* check for replay of PK-encrypted portion. */
+  keylen = crypto_pk_keysize(intro_key);
+  replay = replaycache_add_test_and_elapsed(
+    intro_point->accepted_intro_rsa_parts,
+    parsed_req->ciphertext, MIN(parsed_req->ciphertext_len, keylen),
+    &elapsed);
+
+  if (replay) {
+    log_warn(LD_REND,
+             "Possible replay detected! We received an "
+             "INTRODUCE2 cell with same PK-encrypted part %d "
+             "seconds ago.  Dropping cell.",
+             (int)elapsed);
+    goto err;
+  }
+
+  stage_descr = "decryption";
+  /* Now try to decrypt it */
+  result = rend_service_decrypt_intro(parsed_req, intro_key, &err_msg);
+  if (result < 0) {
+    goto log_error;
+  } else if (err_msg) {
+    log_info(LD_REND, "%s on circ %u.", err_msg,
+             (unsigned)circuit->base_.n_circ_id);
+    tor_free(err_msg);
+  }
+
+  stage_descr = "late parsing";
+  /* Parse the plaintext */
+  result = rend_service_parse_intro_plaintext(parsed_req, &err_msg);
+  if (result < 0) {
+    goto log_error;
+  } else if (err_msg) {
+    log_info(LD_REND, "%s on circ %u.", err_msg,
+             (unsigned)circuit->base_.n_circ_id);
+    tor_free(err_msg);
+  }
+
+  stage_descr = "late validation";
+  /* Validate the parsed plaintext parts */
+  result = rend_service_validate_intro_late(parsed_req, &err_msg);
+  if (result < 0) {
+    goto log_error;
+  } else if (err_msg) {
+    log_info(LD_REND, "%s on circ %u.", err_msg,
+             (unsigned)circuit->base_.n_circ_id);
+    tor_free(err_msg);
+  }
+  stage_descr = NULL;
+
+  /* Increment INTRODUCE2 counter */
+  ++(intro_point->accepted_introduce2_count);
+
+  /* Find the rendezvous point */
+  rp = find_rp_for_intro(parsed_req, &err_msg);
+  if (!rp) {
+    err_msg_severity = LOG_PROTOCOL_WARN;
+    goto log_error;
+  }
+
+  /* Check if we'd refuse to talk to this router */
+  if (options->StrictNodes &&
+      routerset_contains_extendinfo(options->ExcludeNodes, rp)) {
+    log_warn(LD_REND, "Client asked to rendezvous at a relay that we "
+             "exclude, and StrictNodes is set. Refusing service.");
+    reason = END_CIRC_REASON_INTERNAL; /* XXX might leak why we refused */
+    goto err;
+  }
+
+  base16_encode(hexcookie, 9, (const char *)(parsed_req->rc), 4);
+
+  /* Check whether there is a past request with the same Diffie-Hellman,
+   * part 1. */
+  replay = replaycache_add_test_and_elapsed(
+      service->accepted_intro_dh_parts,
+      parsed_req->dh, DH1024_KEY_LEN,
+      &elapsed);
+
+  if (replay) {
+    /* A Tor client will send a new INTRODUCE1 cell with the same rend
+     * cookie and DH public key as its previous one if its intro circ
+     * times out while in state CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT .
+     * If we received the first INTRODUCE1 cell (the intro-point relay
+     * converts it into an INTRODUCE2 cell), we are already trying to
+     * connect to that rend point (and may have already succeeded);
+     * drop this cell. */
+    log_info(LD_REND, "We received an "
+             "INTRODUCE2 cell with same first part of "
+             "Diffie-Hellman handshake %d seconds ago. Dropping "
+             "cell.",
+             (int) elapsed);
+    goto err;
+  }
+
+  /* If the service performs client authorization, check included auth data. */
+  if (service->clients) {
+    if (parsed_req->version == 3 && parsed_req->u.v3.auth_len > 0) {
+      if (rend_check_authorization(service,
+                                   (const char*)parsed_req->u.v3.auth_data,
+                                   parsed_req->u.v3.auth_len)) {
+        log_info(LD_REND, "Authorization data in INTRODUCE2 cell are valid.");
+      } else {
+        log_info(LD_REND, "The authorization data that are contained in "
+                 "the INTRODUCE2 cell are invalid. Dropping cell.");
+        reason = END_CIRC_REASON_CONNECTFAILED;
+        goto err;
+      }
+    } else {
+      log_info(LD_REND, "INTRODUCE2 cell does not contain authentication "
+               "data, but we require client authorization. Dropping cell.");
+      reason = END_CIRC_REASON_CONNECTFAILED;
+      goto err;
+    }
+  }
+
+  /* Try DH handshake... */
+  dh = crypto_dh_new(DH_TYPE_REND);
+  if (!dh || crypto_dh_generate_public(dh)<0) {
+    log_warn(LD_BUG,"Internal error: couldn't build DH state "
+             "or generate public key.");
+    reason = END_CIRC_REASON_INTERNAL;
+    goto err;
+  }
+  if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh,
+                               (char *)(parsed_req->dh),
+                               DH1024_KEY_LEN, keys,
+                               DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) {
+    log_warn(LD_BUG, "Internal error: couldn't complete DH handshake");
+    reason = END_CIRC_REASON_INTERNAL;
+    goto err;
+  }
+
+  circ_needs_uptime = hs_service_requires_uptime_circ(service->ports);
+
+  /* help predict this next time */
+  rep_hist_note_used_internal(now, circ_needs_uptime, 1);
+
+  /* Launch a circuit to the client's chosen rendezvous point.
+   */
+  int max_rend_failures=hs_get_service_max_rend_failures();
+  for (i=0;i<max_rend_failures;i++) {
+    int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
+    if (circ_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME;
+    /* A Single Onion Service only uses a direct connection if its
+     * firewall rules permit direct connections to the address. */
+    if (rend_service_use_direct_connection(options, rp)) {
+      flags = flags | CIRCLAUNCH_ONEHOP_TUNNEL;
+    }
+    launched = circuit_launch_by_extend_info(
+                        CIRCUIT_PURPOSE_S_CONNECT_REND, rp, flags);
+
+    if (launched)
+      break;
+  }
+  if (!launched) { /* give up */
+    log_warn(LD_REND, "Giving up launching first hop of circuit to rendezvous "
+             "point %s for service %s.",
+             safe_str_client(extend_info_describe(rp)),
+             serviceid);
+    reason = END_CIRC_REASON_CONNECTFAILED;
+    goto err;
+  }
+  log_info(LD_REND,
+           "Accepted intro; launching circuit to %s "
+           "(cookie %s) for service %s.",
+           safe_str_client(extend_info_describe(rp)),
+           hexcookie, serviceid);
+  tor_assert(launched->build_state);
+  /* Fill in the circuit's state. */
+
+  launched->rend_data =
+    rend_data_service_create(service->service_id, rend_pk_digest,
+                             parsed_req->rc, service->auth_type);
+
+  launched->build_state->service_pending_final_cpath_ref =
+    tor_malloc_zero(sizeof(crypt_path_reference_t));
+  launched->build_state->service_pending_final_cpath_ref->refcount = 1;
+
+  launched->build_state->service_pending_final_cpath_ref->cpath = cpath =
+    tor_malloc_zero(sizeof(crypt_path_t));
+  cpath->magic = CRYPT_PATH_MAGIC;
+  launched->build_state->expiry_time = now + MAX_REND_TIMEOUT;
+
+  cpath->rend_dh_handshake_state = dh;
+  dh = NULL;
+  if (circuit_init_cpath_crypto(cpath,
+                                keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
+                                1, 0)<0)
+    goto err;
+  memcpy(cpath->rend_circ_nonce, keys, DIGEST_LEN);
+
+  goto done;
+
+ log_error:
+  if (!err_msg) {
+    if (stage_descr) {
+      tor_asprintf(&err_msg,
+                   "unknown %s error for INTRODUCE2", stage_descr);
+    } else {
+      err_msg = tor_strdup("unknown error for INTRODUCE2");
+    }
+  }
+
+  log_fn(err_msg_severity, LD_REND, "%s on circ %u", err_msg,
+           (unsigned)circuit->base_.n_circ_id);
+ err:
+  status = -1;
+  if (dh) crypto_dh_free(dh);
+  if (launched) {
+    circuit_mark_for_close(TO_CIRCUIT(launched), reason);
+  }
+  tor_free(err_msg);
+
+ done:
+  memwipe(keys, 0, sizeof(keys));
+  memwipe(buf, 0, sizeof(buf));
+  memwipe(serviceid, 0, sizeof(serviceid));
+  memwipe(hexcookie, 0, sizeof(hexcookie));
+
+  /* Free the parsed cell */
+  rend_service_free_intro(parsed_req);
+
+  /* Free rp */
+  extend_info_free(rp);
+
+  return status;
+}
+
+/** Given a parsed and decrypted INTRODUCE2, find the rendezvous point or
+ * return NULL and an error string if we can't. Return a newly allocated
+ * extend_info_t* for the rendezvous point. */
+static extend_info_t *
+find_rp_for_intro(const rend_intro_cell_t *intro,
+                  char **err_msg_out)
+{
+  extend_info_t *rp = NULL;
+  char *err_msg = NULL;
+  const char *rp_nickname = NULL;
+  const node_t *node = NULL;
+
+  if (!intro) {
+    if (err_msg_out)
+      err_msg = tor_strdup("Bad parameters to find_rp_for_intro()");
+
+    goto err;
+  }
+
+  if (intro->version == 0 || intro->version == 1) {
+    rp_nickname = (const char *)(intro->u.v0_v1.rp);
+
+    node = node_get_by_nickname(rp_nickname, NNF_NO_WARN_UNNAMED);
+    if (!node) {
+      if (err_msg_out) {
+        tor_asprintf(&err_msg,
+                     "Couldn't find router %s named in INTRODUCE2 cell",
+                     escaped_safe_str_client(rp_nickname));
+      }
+
+      goto err;
+    }
+
+    /* Are we in single onion mode? */
+    const int allow_direct = rend_service_allow_non_anonymous_connection(
+                                                                get_options());
+    rp = extend_info_from_node(node, allow_direct);
+    if (!rp) {
+      if (err_msg_out) {
+        tor_asprintf(&err_msg,
+                     "Couldn't build extend_info_t for router %s named "
+                     "in INTRODUCE2 cell",
+                     escaped_safe_str_client(rp_nickname));
+      }
+
+      goto err;
+    }
+  } else if (intro->version == 2) {
+    rp = extend_info_dup(intro->u.v2.extend_info);
+  } else if (intro->version == 3) {
+    rp = extend_info_dup(intro->u.v3.extend_info);
+  } else {
+    if (err_msg_out) {
+      tor_asprintf(&err_msg,
+                   "Unknown version %d in INTRODUCE2 cell",
+                   (int)(intro->version));
+    }
+
+    goto err;
+  }
+
+  /* rp is always set here: extend_info_dup guarantees a non-NULL result, and
+   * the other cases goto err. */
+  tor_assert(rp);
+
+  /* Make sure the RP we are being asked to connect to is _not_ a private
+   * address unless it's allowed. Let's avoid to build a circuit to our
+   * second middle node and fail right after when extending to the RP. */
+  if (!extend_info_addr_is_allowed(&rp->addr)) {
+    if (err_msg_out) {
+      tor_asprintf(&err_msg,
+                   "Relay IP in INTRODUCE2 cell is private address.");
+    }
+    extend_info_free(rp);
+    rp = NULL;
+    goto err;
+  }
+  goto done;
+
+ err:
+  if (err_msg_out)
+    *err_msg_out = err_msg;
+  else
+    tor_free(err_msg);
+
+ done:
+  return rp;
+}
+
+/** Free a parsed INTRODUCE1 or INTRODUCE2 cell that was allocated by
+ * rend_service_parse_intro().
+ */
+void
+rend_service_free_intro_(rend_intro_cell_t *request)
+{
+  if (!request) {
+    return;
+  }
+
+  /* Free ciphertext */
+  tor_free(request->ciphertext);
+  request->ciphertext_len = 0;
+
+  /* Have plaintext? */
+  if (request->plaintext) {
+    /* Zero it out just to be safe */
+    memwipe(request->plaintext, 0, request->plaintext_len);
+    tor_free(request->plaintext);
+    request->plaintext_len = 0;
+  }
+
+  /* Have parsed plaintext? */
+  if (request->parsed) {
+    switch (request->version) {
+      case 0:
+      case 1:
+        /*
+         * Nothing more to do; these formats have no further pointers
+         * in them.
+         */
+        break;
+      case 2:
+        extend_info_free(request->u.v2.extend_info);
+        request->u.v2.extend_info = NULL;
+        break;
+      case 3:
+        if (request->u.v3.auth_data) {
+          memwipe(request->u.v3.auth_data, 0, request->u.v3.auth_len);
+          tor_free(request->u.v3.auth_data);
+        }
+
+        extend_info_free(request->u.v3.extend_info);
+        request->u.v3.extend_info = NULL;
+        break;
+      default:
+        log_info(LD_BUG,
+                 "rend_service_free_intro() saw unknown protocol "
+                 "version %d.",
+                 request->version);
+    }
+  }
+
+  /* Zero it out to make sure sensitive stuff doesn't hang around in memory */
+  memwipe(request, 0, sizeof(*request));
+
+  tor_free(request);
+}
+
+/** Parse an INTRODUCE1 or INTRODUCE2 cell into a newly allocated
+ * rend_intro_cell_t structure.  Free it with rend_service_free_intro()
+ * when finished.  The type parameter should be 1 or 2 to indicate whether
+ * this is INTRODUCE1 or INTRODUCE2.  This parses only the non-encrypted
+ * parts; after this, call rend_service_decrypt_intro() with a key, then
+ * rend_service_parse_intro_plaintext() to finish parsing.  The optional
+ * err_msg_out parameter is set to a string suitable for log output
+ * if parsing fails.  This function does some validation, but only
+ * that which depends solely on the contents of the cell and the
+ * key; it can be unit-tested.  Further validation is done in
+ * rend_service_validate_intro().
+ */
+
+rend_intro_cell_t *
+rend_service_begin_parse_intro(const uint8_t *request,
+                               size_t request_len,
+                               uint8_t type,
+                               char **err_msg_out)
+{
+  rend_intro_cell_t *rv = NULL;
+  char *err_msg = NULL;
+
+  if (!request || request_len <= 0) goto err;
+  if (!(type == 1 || type == 2)) goto err;
+
+  /* First, check that the cell is long enough to be a sensible INTRODUCE */
+
+  /* min key length plus digest length plus nickname length */
+  if (request_len <
+        (DIGEST_LEN + REND_COOKIE_LEN + (MAX_NICKNAME_LEN + 1) +
+         DH1024_KEY_LEN + 42)) {
+    if (err_msg_out) {
+      tor_asprintf(&err_msg,
+                   "got a truncated INTRODUCE%d cell",
+                   (int)type);
+    }
+    goto err;
+  }
+
+  /* Allocate a new parsed cell structure */
+  rv = tor_malloc_zero(sizeof(*rv));
+
+  /* Set the type */
+  rv->type = type;
+
+  /* Copy in the ID */
+  memcpy(rv->pk, request, DIGEST_LEN);
+
+  /* Copy in the ciphertext */
+  rv->ciphertext = tor_malloc(request_len - DIGEST_LEN);
+  memcpy(rv->ciphertext, request + DIGEST_LEN, request_len - DIGEST_LEN);
+  rv->ciphertext_len = request_len - DIGEST_LEN;
+
+  goto done;
+
+ err:
+  rend_service_free_intro(rv);
+  rv = NULL;
+
+  if (err_msg_out && !err_msg) {
+    tor_asprintf(&err_msg,
+                 "unknown INTRODUCE%d error",
+                 (int)type);
+  }
+
+ done:
+  if (err_msg_out) *err_msg_out = err_msg;
+  else tor_free(err_msg);
+
+  return rv;
+}
+
+/** Parse the version-specific parts of a v0 or v1 INTRODUCE1 or INTRODUCE2
+ * cell
+ */
+
+static ssize_t
+rend_service_parse_intro_for_v0_or_v1(
+    rend_intro_cell_t *intro,
+    const uint8_t *buf,
+    size_t plaintext_len,
+    char **err_msg_out)
+{
+  const char *rp_nickname, *endptr;
+  size_t nickname_field_len, ver_specific_len;
+
+  if (intro->version == 1) {
+    ver_specific_len = MAX_HEX_NICKNAME_LEN + 2;
+    rp_nickname = ((const char *)buf) + 1;
+    nickname_field_len = MAX_HEX_NICKNAME_LEN + 1;
+  } else if (intro->version == 0) {
+    ver_specific_len = MAX_NICKNAME_LEN + 1;
+    rp_nickname = (const char *)buf;
+    nickname_field_len = MAX_NICKNAME_LEN + 1;
+  } else {
+    if (err_msg_out)
+      tor_asprintf(err_msg_out,
+                   "rend_service_parse_intro_for_v0_or_v1() called with "
+                   "bad version %d on INTRODUCE%d cell (this is a bug)",
+                   intro->version,
+                   (int)(intro->type));
+    goto err;
+  }
+
+  if (plaintext_len < ver_specific_len) {
+    if (err_msg_out)
+      tor_asprintf(err_msg_out,
+                   "short plaintext of encrypted part in v1 INTRODUCE%d "
+                   "cell (%lu bytes, needed %lu)",
+                   (int)(intro->type),
+                   (unsigned long)plaintext_len,
+                   (unsigned long)ver_specific_len);
+    goto err;
+  }
+
+  endptr = memchr(rp_nickname, 0, nickname_field_len);
+  if (!endptr || endptr == rp_nickname) {
+    if (err_msg_out) {
+      tor_asprintf(err_msg_out,
+                   "couldn't find a nul-padded nickname in "
+                   "INTRODUCE%d cell",
+                   (int)(intro->type));
+    }
+    goto err;
+  }
+
+  if ((intro->version == 0 &&
+       !is_legal_nickname(rp_nickname)) ||
+      (intro->version == 1 &&
+       !is_legal_nickname_or_hexdigest(rp_nickname))) {
+    if (err_msg_out) {
+      tor_asprintf(err_msg_out,
+                   "bad nickname in INTRODUCE%d cell",
+                   (int)(intro->type));
+    }
+    goto err;
+  }
+
+  memcpy(intro->u.v0_v1.rp, rp_nickname, endptr - rp_nickname + 1);
+
+  return ver_specific_len;
+
+ err:
+  return -1;
+}
+
+/** Parse the version-specific parts of a v2 INTRODUCE1 or INTRODUCE2 cell
+ */
+
+static ssize_t
+rend_service_parse_intro_for_v2(
+    rend_intro_cell_t *intro,
+    const uint8_t *buf,
+    size_t plaintext_len,
+    char **err_msg_out)
+{
+  unsigned int klen;
+  extend_info_t *extend_info = NULL;
+  ssize_t ver_specific_len;
+
+  /*
+   * We accept version 3 too so that the v3 parser can call this with
+   * an adjusted buffer for the latter part of a v3 cell, which is
+   * identical to a v2 cell.
+   */
+  if (!(intro->version == 2 ||
+        intro->version == 3)) {
+    if (err_msg_out)
+      tor_asprintf(err_msg_out,
+                   "rend_service_parse_intro_for_v2() called with "
+                   "bad version %d on INTRODUCE%d cell (this is a bug)",
+                   intro->version,
+                   (int)(intro->type));
+    goto err;
+  }
+
+  /* 7 == version, IP and port, DIGEST_LEN == id, 2 == key length */
+  if (plaintext_len < 7 + DIGEST_LEN + 2) {
+    if (err_msg_out) {
+      tor_asprintf(err_msg_out,
+                   "truncated plaintext of encrypted parted of "
+                   "version %d INTRODUCE%d cell",
+                   intro->version,
+                   (int)(intro->type));
+    }
+
+    goto err;
+  }
+
+  extend_info = tor_malloc_zero(sizeof(extend_info_t));
+  tor_addr_from_ipv4n(&extend_info->addr, get_uint32(buf + 1));
+  extend_info->port = ntohs(get_uint16(buf + 5));
+  memcpy(extend_info->identity_digest, buf + 7, DIGEST_LEN);
+  extend_info->nickname[0] = '$';
+  base16_encode(extend_info->nickname + 1, sizeof(extend_info->nickname) - 1,
+                extend_info->identity_digest, DIGEST_LEN);
+  klen = ntohs(get_uint16(buf + 7 + DIGEST_LEN));
+
+  /* 7 == version, IP and port, DIGEST_LEN == id, 2 == key length */
+  if (plaintext_len < 7 + DIGEST_LEN + 2 + klen) {
+    if (err_msg_out) {
+      tor_asprintf(err_msg_out,
+                   "truncated plaintext of encrypted parted of "
+                   "version %d INTRODUCE%d cell",
+                   intro->version,
+                   (int)(intro->type));
+    }
+
+    goto err;
+  }
+
+  extend_info->onion_key =
+    crypto_pk_asn1_decode((const char *)(buf + 7 + DIGEST_LEN + 2), klen);
+  if (!extend_info->onion_key) {
+    if (err_msg_out) {
+      tor_asprintf(err_msg_out,
+                   "error decoding onion key in version %d "
+                   "INTRODUCE%d cell",
+                   intro->version,
+                   (intro->type));
+    }
+
+    goto err;
+  }
+  if (128 != crypto_pk_keysize(extend_info->onion_key)) {
+    if (err_msg_out) {
+      tor_asprintf(err_msg_out,
+                   "invalid onion key size in version %d INTRODUCE%d cell",
+                   intro->version,
+                   (intro->type));
+    }
+
+    goto err;
+  }
+
+  ver_specific_len = 7+DIGEST_LEN+2+klen;
+
+  if (intro->version == 2) intro->u.v2.extend_info = extend_info;
+  else intro->u.v3.extend_info = extend_info;
+
+  return ver_specific_len;
+
+ err:
+  extend_info_free(extend_info);
+
+  return -1;
+}
+
+/** Parse the version-specific parts of a v3 INTRODUCE1 or INTRODUCE2 cell
+ */
+
+static ssize_t
+rend_service_parse_intro_for_v3(
+    rend_intro_cell_t *intro,
+    const uint8_t *buf,
+    size_t plaintext_len,
+    char **err_msg_out)
+{
+  ssize_t adjust, v2_ver_specific_len, ts_offset;
+
+  /* This should only be called on v3 cells */
+  if (intro->version != 3) {
+    if (err_msg_out)
+      tor_asprintf(err_msg_out,
+                   "rend_service_parse_intro_for_v3() called with "
+                   "bad version %d on INTRODUCE%d cell (this is a bug)",
+                   intro->version,
+                   (int)(intro->type));
+    goto err;
+  }
+
+  /*
+   * Check that we have at least enough to get auth_len:
+   *
+   * 1 octet for version, 1 for auth_type, 2 for auth_len
+   */
+  if (plaintext_len < 4) {
+    if (err_msg_out) {
+      tor_asprintf(err_msg_out,
+                   "truncated plaintext of encrypted parted of "
+                   "version %d INTRODUCE%d cell",
+                   intro->version,
+                   (int)(intro->type));
+    }
+
+    goto err;
+  }
+
+  /*
+   * The rend_client_send_introduction() function over in rendclient.c is
+   * broken (i.e., fails to match the spec) in such a way that we can't
+   * change it without breaking the protocol.  Specifically, it doesn't
+   * emit auth_len when auth-type is REND_NO_AUTH, so everything is off
+   * by two bytes after that.  Calculate ts_offset and do everything from
+   * the timestamp on relative to that to handle this dain bramage.
+   */
+
+  intro->u.v3.auth_type = buf[1];
+  if (intro->u.v3.auth_type != REND_NO_AUTH) {
+    intro->u.v3.auth_len = ntohs(get_uint16(buf + 2));
+    ts_offset = 4 + intro->u.v3.auth_len;
+  } else {
+    intro->u.v3.auth_len = 0;
+    ts_offset = 2;
+  }
+
+  /* Check that auth len makes sense for this auth type */
+  if (intro->u.v3.auth_type == REND_BASIC_AUTH ||
+      intro->u.v3.auth_type == REND_STEALTH_AUTH) {
+      if (intro->u.v3.auth_len != REND_DESC_COOKIE_LEN) {
+        if (err_msg_out) {
+          tor_asprintf(err_msg_out,
+                       "wrong auth data size %d for INTRODUCE%d cell, "
+                       "should be %d",
+                       (int)(intro->u.v3.auth_len),
+                       (int)(intro->type),
+                       REND_DESC_COOKIE_LEN);
+        }
+
+        goto err;
+      }
+  }
+
+  /* Check that we actually have everything up through the timestamp */
+  if (plaintext_len < (size_t)(ts_offset)+4) {
+    if (err_msg_out) {
+      tor_asprintf(err_msg_out,
+                   "truncated plaintext of encrypted parted of "
+                   "version %d INTRODUCE%d cell",
+                   intro->version,
+                   (int)(intro->type));
+    }
+
+    goto err;
+  }
+
+  if (intro->u.v3.auth_type != REND_NO_AUTH &&
+      intro->u.v3.auth_len > 0) {
+    /* Okay, we can go ahead and copy auth_data */
+    intro->u.v3.auth_data = tor_malloc(intro->u.v3.auth_len);
+    /*
+     * We know we had an auth_len field in this case, so 4 is
+     * always right.
+     */
+    memcpy(intro->u.v3.auth_data, buf + 4, intro->u.v3.auth_len);
+  }
+
+  /*
+   * From here on, the format is as in v2, so we call the v2 parser with
+   * adjusted buffer and length.  We are 4 + ts_offset octets in, but the
+   * v2 parser expects to skip over a version byte at the start, so we
+   * adjust by 3 + ts_offset.
+   */
+  adjust = 3 + ts_offset;
+
+  v2_ver_specific_len =
+    rend_service_parse_intro_for_v2(intro,
+                                    buf + adjust, plaintext_len - adjust,
+                                    err_msg_out);
+
+  /* Success in v2 parser */
+  if (v2_ver_specific_len >= 0) return v2_ver_specific_len + adjust;
+  /* Failure in v2 parser; it will have provided an err_msg */
+  else return v2_ver_specific_len;
+
+ err:
+  return -1;
+}
+
+/** Table of parser functions for version-specific parts of an INTRODUCE2
+ * cell.
+ */
+
+static ssize_t
+  (*intro_version_handlers[])(
+    rend_intro_cell_t *,
+    const uint8_t *,
+    size_t,
+    char **) =
+{ rend_service_parse_intro_for_v0_or_v1,
+  rend_service_parse_intro_for_v0_or_v1,
+  rend_service_parse_intro_for_v2,
+  rend_service_parse_intro_for_v3 };
+
+/** Decrypt the encrypted part of an INTRODUCE1 or INTRODUCE2 cell,
+ * return 0 if successful, or < 0 and write an error message to
+ * *err_msg_out if provided.
+ */
+
+int
+rend_service_decrypt_intro(
+    rend_intro_cell_t *intro,
+    crypto_pk_t *key,
+    char **err_msg_out)
+{
+  char *err_msg = NULL;
+  uint8_t key_digest[DIGEST_LEN];
+  char service_id[REND_SERVICE_ID_LEN_BASE32+1];
+  ssize_t key_len;
+  uint8_t buf[RELAY_PAYLOAD_SIZE];
+  int result, status = -1;
+
+  if (!intro || !key) {
+    if (err_msg_out) {
+      err_msg =
+        tor_strdup("rend_service_decrypt_intro() called with bad "
+                   "parameters");
+    }
+
+    status = -2;
+    goto err;
+  }
+
+  /* Make sure we have ciphertext */
+  if (!(intro->ciphertext) || intro->ciphertext_len <= 0) {
+    if (err_msg_out) {
+      tor_asprintf(&err_msg,
+                   "rend_intro_cell_t was missing ciphertext for "
+                   "INTRODUCE%d cell",
+                   (int)(intro->type));
+    }
+    status = -3;
+    goto err;
+  }
+
+  /* Check that this cell actually matches this service key */
+
+  /* first DIGEST_LEN bytes of request is intro or service pk digest */
+  if (crypto_pk_get_digest(key, (char *)key_digest) < 0) {
+    if (err_msg_out)
+      *err_msg_out = tor_strdup("Couldn't compute RSA digest.");
+    log_warn(LD_BUG, "Couldn't compute key digest.");
+    status = -7;
+    goto err;
+  }
+
+  if (tor_memneq(key_digest, intro->pk, DIGEST_LEN)) {
+    if (err_msg_out) {
+      base32_encode(service_id, REND_SERVICE_ID_LEN_BASE32 + 1,
+                    (char*)(intro->pk), REND_SERVICE_ID_LEN);
+      tor_asprintf(&err_msg,
+                   "got an INTRODUCE%d cell for the wrong service (%s)",
+                   (int)(intro->type),
+                   escaped(service_id));
+    }
+
+    status = -4;
+    goto err;
+  }
+
+  /* Make sure the encrypted part is long enough to decrypt */
+
+  key_len = crypto_pk_keysize(key);
+  if (intro->ciphertext_len < key_len) {
+    if (err_msg_out) {
+      tor_asprintf(&err_msg,
+                   "got an INTRODUCE%d cell with a truncated PK-encrypted "
+                   "part",
+                   (int)(intro->type));
+    }
+
+    status = -5;
+    goto err;
+  }
+
+  /* Decrypt the encrypted part */
+  result =
+    crypto_pk_obsolete_private_hybrid_decrypt(
+       key, (char *)buf, sizeof(buf),
+       (const char *)(intro->ciphertext), intro->ciphertext_len,
+       PK_PKCS1_OAEP_PADDING, 1);
+  if (result < 0) {
+    if (err_msg_out) {
+      tor_asprintf(&err_msg,
+                   "couldn't decrypt INTRODUCE%d cell",
+                   (int)(intro->type));
+    }
+    status = -6;
+    goto err;
+  }
+  intro->plaintext_len = result;
+  intro->plaintext = tor_malloc(intro->plaintext_len);
+  memcpy(intro->plaintext, buf, intro->plaintext_len);
+
+  status = 0;
+
+  goto done;
+
+ err:
+  if (err_msg_out && !err_msg) {
+    tor_asprintf(&err_msg,
+                 "unknown INTRODUCE%d error decrypting encrypted part",
+                 intro ? (int)(intro->type) : -1);
+  }
+
+ done:
+  if (err_msg_out) *err_msg_out = err_msg;
+  else tor_free(err_msg);
+
+  /* clean up potentially sensitive material */
+  memwipe(buf, 0, sizeof(buf));
+  memwipe(key_digest, 0, sizeof(key_digest));
+  memwipe(service_id, 0, sizeof(service_id));
+
+  return status;
+}
+
+/** Parse the plaintext of the encrypted part of an INTRODUCE1 or
+ * INTRODUCE2 cell, return 0 if successful, or < 0 and write an error
+ * message to *err_msg_out if provided.
+ */
+
+int
+rend_service_parse_intro_plaintext(
+    rend_intro_cell_t *intro,
+    char **err_msg_out)
+{
+  char *err_msg = NULL;
+  ssize_t ver_specific_len, ver_invariant_len;
+  uint8_t version;
+  int status = -1;
+
+  if (!intro) {
+    if (err_msg_out) {
+      err_msg =
+        tor_strdup("rend_service_parse_intro_plaintext() called with NULL "
+                   "rend_intro_cell_t");
+    }
+
+    status = -2;
+    goto err;
+  }
+
+  /* Check that we have plaintext */
+  if (!(intro->plaintext) || intro->plaintext_len <= 0) {
+    if (err_msg_out) {
+      err_msg = tor_strdup("rend_intro_cell_t was missing plaintext");
+    }
+    status = -3;
+    goto err;
+  }
+
+  /* In all formats except v0, the first byte is a version number */
+  version = intro->plaintext[0];
+
+  /* v0 has no version byte (stupid...), so handle it as a fallback */
+  if (version > 3) version = 0;
+
+  /* Copy the version into the parsed cell structure */
+  intro->version = version;
+
+  /* Call the version-specific parser from the table */
+  ver_specific_len =
+    intro_version_handlers[version](intro,
+                                    intro->plaintext, intro->plaintext_len,
+                                    &err_msg);
+  if (ver_specific_len < 0) {
+    status = -4;
+    goto err;
+  }
+
+  /** The rendezvous cookie and Diffie-Hellman stuff are version-invariant
+   * and at the end of the plaintext of the encrypted part of the cell.
+   */
+
+  ver_invariant_len = intro->plaintext_len - ver_specific_len;
+  if (ver_invariant_len < REND_COOKIE_LEN + DH1024_KEY_LEN) {
+    tor_asprintf(&err_msg,
+        "decrypted plaintext of INTRODUCE%d cell was truncated (%ld bytes)",
+        (int)(intro->type),
+        (long)(intro->plaintext_len));
+    status = -5;
+    goto err;
+  } else if (ver_invariant_len > REND_COOKIE_LEN + DH1024_KEY_LEN) {
+    tor_asprintf(&err_msg,
+        "decrypted plaintext of INTRODUCE%d cell was too long (%ld bytes)",
+        (int)(intro->type),
+        (long)(intro->plaintext_len));
+    status = -6;
+    goto err;
+  } else {
+    memcpy(intro->rc,
+           intro->plaintext + ver_specific_len,
+           REND_COOKIE_LEN);
+    memcpy(intro->dh,
+           intro->plaintext + ver_specific_len + REND_COOKIE_LEN,
+           DH1024_KEY_LEN);
+  }
+
+  /* Flag it as being fully parsed */
+  intro->parsed = 1;
+
+  status = 0;
+  goto done;
+
+ err:
+  if (err_msg_out && !err_msg) {
+    tor_asprintf(&err_msg,
+                 "unknown INTRODUCE%d error parsing encrypted part",
+                 intro ? (int)(intro->type) : -1);
+  }
+
+ done:
+  if (err_msg_out) *err_msg_out = err_msg;
+  else tor_free(err_msg);
+
+  return status;
+}
+
+/** Do validity checks on a parsed intro cell after decryption; some of
+ * these are not done in rend_service_parse_intro_plaintext() itself because
+ * they depend on a lot of other state and would make it hard to unit test.
+ * Returns >= 0 if successful or < 0 if the intro cell is invalid, and
+ * optionally writes out an error message for logging.  If an err_msg
+ * pointer is provided, it is the caller's responsibility to free any
+ * provided message.
+ */
+
+int
+rend_service_validate_intro_late(const rend_intro_cell_t *intro,
+                                 char **err_msg_out)
+{
+  int status = 0;
+
+  if (!intro) {
+    if (err_msg_out)
+      *err_msg_out =
+        tor_strdup("NULL intro cell passed to "
+                   "rend_service_validate_intro_late()");
+
+    status = -1;
+    goto err;
+  }
+
+  if (intro->version == 3 && intro->parsed) {
+    if (!(intro->u.v3.auth_type == REND_NO_AUTH ||
+          intro->u.v3.auth_type == REND_BASIC_AUTH ||
+          intro->u.v3.auth_type == REND_STEALTH_AUTH)) {
+      /* This is an informative message, not an error, as in the old code */
+      if (err_msg_out)
+        tor_asprintf(err_msg_out,
+                     "unknown authorization type %d",
+                     intro->u.v3.auth_type);
+    }
+  }
+
+ err:
+  return status;
+}
+
+/** Called when we fail building a rendezvous circuit at some point other
+ * than the last hop: launches a new circuit to the same rendezvous point.
+ */
+void
+rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
+{
+  origin_circuit_t *newcirc;
+  cpath_build_state_t *newstate, *oldstate;
+
+  tor_assert(oldcirc->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+  oldstate = oldcirc->build_state;
+  tor_assert(oldstate);
+
+  if (oldstate->service_pending_final_cpath_ref == NULL) {
+    log_info(LD_REND,"Skipping relaunch of circ that failed on its first hop. "
+             "Initiator will retry.");
+    return;
+  }
+
+  log_info(LD_REND,"Reattempting rendezvous circuit to '%s'",
+           safe_str(extend_info_describe(oldstate->chosen_exit)));
+
+  /* You'd think Single Onion Services would want to retry the rendezvous
+   * using a direct connection. But if it's blocked by a firewall, or the
+   * service is IPv6-only, or the rend point avoiding becoming a one-hop
+   * proxy, we need a 3-hop connection. */
+  newcirc = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND,
+                            oldstate->chosen_exit,
+                            CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
+
+  if (!newcirc) {
+    log_warn(LD_REND,"Couldn't relaunch rendezvous circuit to '%s'.",
+             safe_str(extend_info_describe(oldstate->chosen_exit)));
+    return;
+  }
+  newstate = newcirc->build_state;
+  tor_assert(newstate);
+  newstate->failure_count = oldstate->failure_count+1;
+  newstate->expiry_time = oldstate->expiry_time;
+  newstate->service_pending_final_cpath_ref =
+    oldstate->service_pending_final_cpath_ref;
+  ++(newstate->service_pending_final_cpath_ref->refcount);
+
+  newcirc->rend_data = rend_data_dup(oldcirc->rend_data);
+}
+
+/** Launch a circuit to serve as an introduction point for the service
+ * <b>service</b> at the introduction point <b>nickname</b>
+ */
+static int
+rend_service_launch_establish_intro(rend_service_t *service,
+                                    rend_intro_point_t *intro)
+{
+  origin_circuit_t *launched;
+  int flags = CIRCLAUNCH_NEED_UPTIME|CIRCLAUNCH_IS_INTERNAL;
+  const or_options_t *options = get_options();
+  extend_info_t *launch_ei = intro->extend_info;
+  extend_info_t *direct_ei = NULL;
+
+  /* Are we in single onion mode? */
+  if (rend_service_allow_non_anonymous_connection(options)) {
+    /* Do we have a descriptor for the node?
+     * We've either just chosen it from the consensus, or we've just reviewed
+     * our intro points to see which ones are still valid, and deleted the ones
+     * that aren't in the consensus any more. */
+    const node_t *node = node_get_by_id(launch_ei->identity_digest);
+    if (BUG(!node)) {
+      /* The service has kept an intro point after it went missing from the
+       * consensus. If we did anything else here, it would be a consensus
+       * distinguisher. Which are less of an issue for single onion services,
+       * but still a bug. */
+      return -1;
+    }
+    /* Can we connect to the node directly? If so, replace launch_ei
+     * (a multi-hop extend_info) with one suitable for direct connection. */
+    if (rend_service_use_direct_connection_node(options, node)) {
+      direct_ei = extend_info_from_node(node, 1);
+      if (BUG(!direct_ei)) {
+        /* rend_service_use_direct_connection_node and extend_info_from_node
+         * disagree about which addresses on this node are permitted. This
+         * should never happen. Avoiding the connection is a safe response. */
+        return -1;
+      }
+      flags = flags | CIRCLAUNCH_ONEHOP_TUNNEL;
+      launch_ei = direct_ei;
+    }
+  }
+  /* launch_ei is either intro->extend_info, or has been replaced with a valid
+   * extend_info for single onion service direct connection. */
+  tor_assert(launch_ei);
+  /* We must have the same intro when making a direct connection. */
+  tor_assert(tor_memeq(intro->extend_info->identity_digest,
+                       launch_ei->identity_digest,
+                       DIGEST_LEN));
+
+  log_info(LD_REND,
+           "Launching circuit to introduction point %s%s%s for service %s",
+           safe_str_client(extend_info_describe(intro->extend_info)),
+           direct_ei ? " via direct address " : "",
+           direct_ei ? safe_str_client(extend_info_describe(direct_ei)) : "",
+           service->service_id);
+
+  rep_hist_note_used_internal(time(NULL), 1, 0);
+
+  ++service->n_intro_circuits_launched;
+  launched = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO,
+                             launch_ei, flags);
+
+  if (!launched) {
+    log_info(LD_REND,
+             "Can't launch circuit to establish introduction at %s%s%s.",
+             safe_str_client(extend_info_describe(intro->extend_info)),
+             direct_ei ? " via direct address " : "",
+             direct_ei ? safe_str_client(extend_info_describe(direct_ei)) : ""
+             );
+    extend_info_free(direct_ei);
+    return -1;
+  }
+  /* We must have the same exit node even if cannibalized or direct connection.
+   */
+  tor_assert(tor_memeq(intro->extend_info->identity_digest,
+                       launched->build_state->chosen_exit->identity_digest,
+                       DIGEST_LEN));
+
+  launched->rend_data = rend_data_service_create(service->service_id,
+                                                 service->pk_digest, NULL,
+                                                 service->auth_type);
+  launched->intro_key = crypto_pk_dup_key(intro->intro_key);
+  if (launched->base_.state == CIRCUIT_STATE_OPEN)
+    rend_service_intro_has_opened(launched);
+  extend_info_free(direct_ei);
+  return 0;
+}
+
+/** Return the number of introduction points that are established for the
+ * given service. */
+static unsigned int
+count_established_intro_points(const rend_service_t *service)
+{
+  unsigned int num = 0;
+
+  SMARTLIST_FOREACH(service->intro_nodes, rend_intro_point_t *, intro,
+    num += intro->circuit_established
+  );
+  return num;
+}
+
+/** Return the number of introduction points that are or are being
+ * established for the given service. This function iterates over all
+ * circuit and count those that are linked to the service and are waiting
+ * for the intro point to respond. */
+static unsigned int
+count_intro_point_circuits(const rend_service_t *service)
+{
+  unsigned int num_ipos = 0;
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!circ->marked_for_close &&
+        circ->state == CIRCUIT_STATE_OPEN &&
+        (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+         circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
+      origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+      if (oc->rend_data &&
+          rend_circuit_pk_digest_eq(oc, (uint8_t *) service->pk_digest)) {
+        num_ipos++;
+      }
+    }
+  }
+  SMARTLIST_FOREACH_END(circ);
+  return num_ipos;
+}
+
+/* Given a buffer of at least RELAY_PAYLOAD_SIZE bytes in <b>cell_body_out</b>,
+   write the body of a legacy ESTABLISH_INTRO cell in it. Use <b>intro_key</b>
+   as the intro point auth key, and <b>rend_circ_nonce</b> as the circuit
+   crypto material. On success, fill <b>cell_body_out</b> and return the number
+   of bytes written. On fail, return -1.
+ */
+ssize_t
+rend_service_encode_establish_intro_cell(char *cell_body_out,
+                                         size_t cell_body_out_len,
+                                         crypto_pk_t *intro_key,
+                                         const char *rend_circ_nonce)
+{
+  int retval = -1;
+  int r;
+  int len = 0;
+  char auth[DIGEST_LEN + 9];
+
+  tor_assert(intro_key);
+  tor_assert(rend_circ_nonce);
+
+  /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */
+  r = crypto_pk_asn1_encode(intro_key, cell_body_out+2,
+                            RELAY_PAYLOAD_SIZE-2);
+  if (r < 0) {
+    log_warn(LD_BUG, "Internal error; failed to establish intro point.");
+    goto err;
+  }
+  len = r;
+  set_uint16(cell_body_out, htons((uint16_t)len));
+  len += 2;
+  memcpy(auth, rend_circ_nonce, DIGEST_LEN);
+  memcpy(auth+DIGEST_LEN, "INTRODUCE", 9);
+  if (crypto_digest(cell_body_out+len, auth, DIGEST_LEN+9))
+    goto err;
+  len += 20;
+  r = crypto_pk_private_sign_digest(intro_key, cell_body_out+len,
+                                    cell_body_out_len - len,
+                                    cell_body_out, len);
+  if (r<0) {
+    log_warn(LD_BUG, "Internal error: couldn't sign introduction request.");
+    goto err;
+  }
+  len += r;
+
+  retval = len;
+
+ err:
+  memwipe(auth, 0, sizeof(auth));
+
+  return retval;
+}
+
+/** Called when we're done building a circuit to an introduction point:
+ *  sends a RELAY_ESTABLISH_INTRO cell.
+ */
+void
+rend_service_intro_has_opened(origin_circuit_t *circuit)
+{
+  rend_service_t *service;
+  char buf[RELAY_PAYLOAD_SIZE];
+  char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  unsigned int expiring_nodes_len, num_ip_circuits, valid_ip_circuits = 0;
+  int reason = END_CIRC_REASON_TORPROTOCOL;
+  const char *rend_pk_digest;
+
+  tor_assert(circuit->base_.purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+  assert_circ_anonymity_ok(circuit, get_options());
+  tor_assert(circuit->cpath);
+  tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only on supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
+
+  base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                rend_pk_digest, REND_SERVICE_ID_LEN);
+
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
+  if (!service) {
+    log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %u.",
+             safe_str_client(serviceid), (unsigned)circuit->base_.n_circ_id);
+    reason = END_CIRC_REASON_NOSUCHSERVICE;
+    goto err;
+  }
+
+  /* Take the current amount of expiring nodes and the current amount of IP
+   * circuits and compute how many valid IP circuits we have. */
+  expiring_nodes_len = (unsigned int) smartlist_len(service->expiring_nodes);
+  num_ip_circuits = count_intro_point_circuits(service);
+  /* Let's avoid an underflow. The valid_ip_circuits is initialized to 0 in
+   * case this condition turns out false because it means that all circuits
+   * are expiring so we need to keep this circuit. */
+  if (num_ip_circuits > expiring_nodes_len) {
+    valid_ip_circuits = num_ip_circuits - expiring_nodes_len;
+  }
+
+  /* If we already have enough introduction circuits for this service,
+   * redefine this one as a general circuit or close it, depending.
+   * Subtract the amount of expiring nodes here because the circuits are
+   * still opened. */
+  if (valid_ip_circuits > service->n_intro_points_wanted) {
+    const or_options_t *options = get_options();
+    /* Remove the intro point associated with this circuit, it's being
+     * repurposed or closed thus cleanup memory. */
+    rend_intro_point_t *intro = find_intro_point(circuit);
+    if (intro != NULL) {
+      smartlist_remove(service->intro_nodes, intro);
+      rend_intro_point_free(intro);
+    }
+
+    if (options->ExcludeNodes) {
+      /* XXXX in some future version, we can test whether the transition is
+         allowed or not given the actual nodes in the circuit.  But for now,
+         this case, we might as well close the thing. */
+      log_info(LD_CIRC|LD_REND, "We have just finished an introduction "
+               "circuit, but we already have enough.  Closing it.");
+      reason = END_CIRC_REASON_NONE;
+      goto err;
+    } else {
+      tor_assert(circuit->build_state->is_internal);
+      log_info(LD_CIRC|LD_REND, "We have just finished an introduction "
+               "circuit, but we already have enough. Redefining purpose to "
+               "general; leaving as internal.");
+
+      if (circuit_should_use_vanguards(TO_CIRCUIT(circuit)->purpose)) {
+        circuit_change_purpose(TO_CIRCUIT(circuit),
+                CIRCUIT_PURPOSE_HS_VANGUARDS);
+      } else {
+        circuit_change_purpose(TO_CIRCUIT(circuit), CIRCUIT_PURPOSE_C_GENERAL);
+      }
+
+      {
+        rend_data_free(circuit->rend_data);
+        circuit->rend_data = NULL;
+      }
+      {
+        crypto_pk_t *intro_key = circuit->intro_key;
+        circuit->intro_key = NULL;
+        crypto_pk_free(intro_key);
+      }
+
+      circuit_has_opened(circuit);
+      goto done;
+    }
+  }
+
+  log_info(LD_REND,
+           "Established circuit %u as introduction point for service %s",
+           (unsigned)circuit->base_.n_circ_id, serviceid);
+  circuit_log_path(LOG_INFO, LD_REND, circuit);
+
+  /* Send the ESTABLISH_INTRO cell */
+  {
+    ssize_t len;
+    len = rend_service_encode_establish_intro_cell(buf, sizeof(buf),
+                                      circuit->intro_key,
+                                      circuit->cpath->prev->rend_circ_nonce);
+    if (len < 0) {
+      reason = END_CIRC_REASON_INTERNAL;
+      goto err;
+    }
+
+    if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
+                                     RELAY_COMMAND_ESTABLISH_INTRO,
+                                     buf, len, circuit->cpath->prev)<0) {
+      log_info(LD_GENERAL,
+             "Couldn't send introduction request for service %s on circuit %u",
+             serviceid, (unsigned)circuit->base_.n_circ_id);
+      goto done;
+    }
+  }
+
+  /* We've attempted to use this circuit */
+  pathbias_count_use_attempt(circuit);
+
+  goto done;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circuit), reason);
+ done:
+  memwipe(buf, 0, sizeof(buf));
+  memwipe(serviceid, 0, sizeof(serviceid));
+
+  return;
+}
+
+/** Called when we get an INTRO_ESTABLISHED cell; mark the circuit as a
+ * live introduction point, and note that the service descriptor is
+ * now out-of-date. */
+int
+rend_service_intro_established(origin_circuit_t *circuit,
+                               const uint8_t *request,
+                               size_t request_len)
+{
+  rend_service_t *service;
+  rend_intro_point_t *intro;
+  char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  (void) request;
+  (void) request_len;
+  tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only supported one for now). */
+  const char *rend_pk_digest =
+    (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
+
+  if (circuit->base_.purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) {
+    log_warn(LD_PROTOCOL,
+             "received INTRO_ESTABLISHED cell on non-intro circuit.");
+    goto err;
+  }
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
+  if (!service) {
+    log_warn(LD_REND, "Unknown service on introduction circuit %u.",
+             (unsigned)circuit->base_.n_circ_id);
+    goto err;
+  }
+  base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32 + 1,
+                rend_pk_digest, REND_SERVICE_ID_LEN);
+  /* We've just successfully established a intro circuit to one of our
+   * introduction point, account for it. */
+  intro = find_intro_point(circuit);
+  if (intro == NULL) {
+    log_warn(LD_REND,
+             "Introduction circuit established without a rend_intro_point_t "
+             "object for service %s on circuit %u",
+             safe_str_client(serviceid), (unsigned)circuit->base_.n_circ_id);
+    goto err;
+  }
+  intro->circuit_established = 1;
+  /* We might not have every introduction point ready but at this point we
+   * know that the descriptor needs to be uploaded. */
+  service->desc_is_dirty = time(NULL);
+  circuit_change_purpose(TO_CIRCUIT(circuit), CIRCUIT_PURPOSE_S_INTRO);
+
+  log_info(LD_REND,
+           "Received INTRO_ESTABLISHED cell on circuit %u for service %s",
+           (unsigned)circuit->base_.n_circ_id, serviceid);
+
+  /* Getting a valid INTRODUCE_ESTABLISHED means we've successfully
+   * used the circ */
+  pathbias_mark_use_success(circuit);
+
+  return 0;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circuit), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/** Called once a circuit to a rendezvous point is established: sends a
+ *  RELAY_COMMAND_RENDEZVOUS1 cell.
+ */
+void
+rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
+{
+  rend_service_t *service;
+  char buf[RELAY_PAYLOAD_SIZE];
+  crypt_path_t *hop;
+  char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  char hexcookie[9];
+  int reason;
+  const char *rend_cookie, *rend_pk_digest;
+
+  tor_assert(circuit->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+  tor_assert(circuit->cpath);
+  tor_assert(circuit->build_state);
+  assert_circ_anonymity_ok(circuit, get_options());
+  tor_assert(circuit->rend_data);
+
+  /* XXX: This is version 2 specific (only one supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data,
+                                                    NULL);
+  rend_cookie = circuit->rend_data->rend_cookie;
+
+  /* Declare the circuit dirty to avoid reuse, and for path-bias. We set the
+   * timestamp regardless of its content because that circuit could have been
+   * cannibalized so in any cases, we are about to use that circuit more. */
+  circuit->base_.timestamp_dirty = time(NULL);
+
+  /* This may be redundant */
+  pathbias_count_use_attempt(circuit);
+
+  hop = circuit->build_state->service_pending_final_cpath_ref->cpath;
+
+  base16_encode(hexcookie,9, rend_cookie,4);
+  base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                rend_pk_digest, REND_SERVICE_ID_LEN);
+
+  log_info(LD_REND,
+           "Done building circuit %u to rendezvous with "
+           "cookie %s for service %s",
+           (unsigned)circuit->base_.n_circ_id, hexcookie, serviceid);
+  circuit_log_path(LOG_INFO, LD_REND, circuit);
+
+  /* Clear the 'in-progress HS circ has timed out' flag for
+   * consistency with what happens on the client side; this line has
+   * no effect on Tor's behaviour. */
+  circuit->hs_circ_has_timed_out = 0;
+
+  /* If hop is NULL, another rend circ has already connected to this
+   * rend point.  Close this circ. */
+  if (hop == NULL) {
+    log_info(LD_REND, "Another rend circ has already reached this rend point; "
+             "closing this rend circ.");
+    reason = END_CIRC_REASON_NONE;
+    goto err;
+  }
+
+  /* Remove our final cpath element from the reference, so that no
+   * other circuit will try to use it.  Store it in
+   * pending_final_cpath for now to ensure that it will be freed if
+   * our rendezvous attempt fails. */
+  circuit->build_state->pending_final_cpath = hop;
+  circuit->build_state->service_pending_final_cpath_ref->cpath = NULL;
+
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
+  if (!service) {
+    log_warn(LD_GENERAL, "Internal error: unrecognized service ID on "
+             "rendezvous circuit.");
+    reason = END_CIRC_REASON_INTERNAL;
+    goto err;
+  }
+
+  /* All we need to do is send a RELAY_RENDEZVOUS1 cell... */
+  memcpy(buf, rend_cookie, REND_COOKIE_LEN);
+  if (crypto_dh_get_public(hop->rend_dh_handshake_state,
+                           buf+REND_COOKIE_LEN, DH1024_KEY_LEN)<0) {
+    log_warn(LD_GENERAL,"Couldn't get DH public key.");
+    reason = END_CIRC_REASON_INTERNAL;
+    goto err;
+  }
+  memcpy(buf+REND_COOKIE_LEN+DH1024_KEY_LEN, hop->rend_circ_nonce,
+         DIGEST_LEN);
+
+  /* Send the cell */
+  if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
+                                   RELAY_COMMAND_RENDEZVOUS1,
+                                   buf, HS_LEGACY_RENDEZVOUS_CELL_SIZE,
+                                   circuit->cpath->prev)<0) {
+    log_warn(LD_GENERAL, "Couldn't send RENDEZVOUS1 cell.");
+    goto done;
+  }
+
+  crypto_dh_free(hop->rend_dh_handshake_state);
+  hop->rend_dh_handshake_state = NULL;
+
+  /* Append the cpath entry. */
+  hop->state = CPATH_STATE_OPEN;
+  /* set the windows to default. these are the windows
+   * that the service thinks the client has.
+   */
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  onion_append_to_cpath(&circuit->cpath, hop);
+  circuit->build_state->pending_final_cpath = NULL; /* prevent double-free */
+
+  /* Change the circuit purpose. */
+  circuit_change_purpose(TO_CIRCUIT(circuit), CIRCUIT_PURPOSE_S_REND_JOINED);
+
+  goto done;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circuit), reason);
+ done:
+  memwipe(buf, 0, sizeof(buf));
+  memwipe(serviceid, 0, sizeof(serviceid));
+  memwipe(hexcookie, 0, sizeof(hexcookie));
+
+  return;
+}
+
+/*
+ * Manage introduction points
+ */
+
+/** Return the (possibly non-open) introduction circuit ending at
+ * <b>intro</b> for the service whose public key is <b>pk_digest</b>.
+ * (<b>desc_version</b> is ignored). Return NULL if no such service is
+ * found.
+ */
+static origin_circuit_t *
+find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest)
+{
+  origin_circuit_t *circ = NULL;
+
+  tor_assert(intro);
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ,
+                         (uint8_t *) pk_digest, CIRCUIT_PURPOSE_S_INTRO))) {
+    if (tor_memeq(circ->build_state->chosen_exit->identity_digest,
+                intro->extend_info->identity_digest, DIGEST_LEN) &&
+        circ->rend_data) {
+      return circ;
+    }
+  }
+
+  circ = NULL;
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ,
+                         (uint8_t *) pk_digest,
+                         CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) {
+    if (tor_memeq(circ->build_state->chosen_exit->identity_digest,
+                intro->extend_info->identity_digest, DIGEST_LEN) &&
+        circ->rend_data) {
+      return circ;
+    }
+  }
+  return NULL;
+}
+
+/** Return the corresponding introdution point using the circuit <b>circ</b>
+ * found in the <b>service</b>. NULL is returned if not found. */
+static rend_intro_point_t *
+find_expiring_intro_point(rend_service_t *service, origin_circuit_t *circ)
+{
+  tor_assert(service);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+             TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO);
+
+  SMARTLIST_FOREACH(service->expiring_nodes, rend_intro_point_t *,
+                    intro_point,
+    if (crypto_pk_eq_keys(intro_point->intro_key, circ->intro_key)) {
+      return intro_point;
+  });
+
+  return NULL;
+}
+
+/** Return a pointer to the rend_intro_point_t corresponding to the
+ * service-side introduction circuit <b>circ</b>. */
+static rend_intro_point_t *
+find_intro_point(origin_circuit_t *circ)
+{
+  const char *serviceid;
+  rend_service_t *service = NULL;
+
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+             TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO);
+  tor_assert(circ->rend_data);
+  serviceid = rend_data_get_address(circ->rend_data);
+
+  SMARTLIST_FOREACH(rend_service_list, rend_service_t *, s,
+    if (tor_memeq(s->service_id, serviceid, REND_SERVICE_ID_LEN_BASE32)) {
+      service = s;
+      break;
+    });
+
+  if (service == NULL) return NULL;
+
+  SMARTLIST_FOREACH(service->intro_nodes, rend_intro_point_t *, intro_point,
+    if (crypto_pk_eq_keys(intro_point->intro_key, circ->intro_key)) {
+      return intro_point;
+    });
+
+  return NULL;
+}
+
+/** Upload the rend_encoded_v2_service_descriptor_t's in <b>descs</b>
+ * associated with the rend_service_descriptor_t <b>renddesc</b> to
+ * the responsible hidden service directories OR the hidden service
+ * directories specified by <b>hs_dirs</b>; <b>service_id</b> and
+ * <b>seconds_valid</b> are only passed for logging purposes.
+ */
+void
+directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
+                         smartlist_t *descs, smartlist_t *hs_dirs,
+                         const char *service_id, int seconds_valid)
+{
+  int i, j, failed_upload = 0;
+  smartlist_t *responsible_dirs = smartlist_new();
+  smartlist_t *successful_uploads = smartlist_new();
+  routerstatus_t *hs_dir;
+  for (i = 0; i < smartlist_len(descs); i++) {
+    rend_encoded_v2_service_descriptor_t *desc = smartlist_get(descs, i);
+    /** If any HSDirs are specified, they should be used instead of
+     *  the responsible directories */
+    if (hs_dirs && smartlist_len(hs_dirs) > 0) {
+      smartlist_add_all(responsible_dirs, hs_dirs);
+    } else {
+      /* Determine responsible dirs. */
+      if (hid_serv_get_responsible_directories(responsible_dirs,
+                                               desc->desc_id) < 0) {
+        log_warn(LD_REND, "Could not determine the responsible hidden service "
+                          "directories to post descriptors to.");
+        control_event_hs_descriptor_upload(service_id,
+                                           "UNKNOWN",
+                                           "UNKNOWN", NULL);
+        goto done;
+      }
+    }
+    for (j = 0; j < smartlist_len(responsible_dirs); j++) {
+      char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+      char *hs_dir_ip;
+      const node_t *node;
+      rend_data_t *rend_data;
+      hs_dir = smartlist_get(responsible_dirs, j);
+      if (smartlist_contains_digest(renddesc->successful_uploads,
+                                hs_dir->identity_digest))
+        /* Don't upload descriptor if we succeeded in doing so last time. */
+        continue;
+      node = node_get_by_id(hs_dir->identity_digest);
+      if (!node || !node_has_preferred_descriptor(node,0)) {
+        log_info(LD_REND, "Not launching upload for for v2 descriptor to "
+                          "hidden service directory %s; we don't have its "
+                          "router descriptor. Queuing for later upload.",
+                 safe_str_client(routerstatus_describe(hs_dir)));
+        failed_upload = -1;
+        continue;
+      }
+      /* Send publish request. */
+
+      /* We need the service ID to identify which service did the upload
+       * request. Lookup is made in rend_service_desc_has_uploaded(). */
+      rend_data = rend_data_client_create(service_id, desc->desc_id, NULL,
+                                          REND_NO_AUTH);
+      directory_request_t *req =
+        directory_request_new(DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+      directory_request_set_routerstatus(req, hs_dir);
+      directory_request_set_indirection(req, DIRIND_ANONYMOUS);
+      directory_request_set_payload(req,
+                                    desc->desc_str, strlen(desc->desc_str));
+      directory_request_set_rend_query(req, rend_data);
+      directory_initiate_request(req);
+      directory_request_free(req);
+
+      rend_data_free(rend_data);
+      base32_encode(desc_id_base32, sizeof(desc_id_base32),
+                    desc->desc_id, DIGEST_LEN);
+      hs_dir_ip = tor_dup_ip(hs_dir->addr);
+      log_info(LD_REND, "Launching upload for v2 descriptor for "
+                        "service '%s' with descriptor ID '%s' with validity "
+                        "of %d seconds to hidden service directory '%s' on "
+                        "%s:%d.",
+               safe_str_client(service_id),
+               safe_str_client(desc_id_base32),
+               seconds_valid,
+               hs_dir->nickname,
+               hs_dir_ip,
+               hs_dir->or_port);
+      control_event_hs_descriptor_upload(service_id,
+                                         hs_dir->identity_digest,
+                                         desc_id_base32, NULL);
+      tor_free(hs_dir_ip);
+      /* Remember successful upload to this router for next time. */
+      if (!smartlist_contains_digest(successful_uploads,
+                                     hs_dir->identity_digest))
+        smartlist_add(successful_uploads, hs_dir->identity_digest);
+    }
+    smartlist_clear(responsible_dirs);
+  }
+  if (!failed_upload) {
+    if (renddesc->successful_uploads) {
+      SMARTLIST_FOREACH(renddesc->successful_uploads, char *, c, tor_free(c););
+      smartlist_free(renddesc->successful_uploads);
+      renddesc->successful_uploads = NULL;
+    }
+    renddesc->all_uploads_performed = 1;
+  } else {
+    /* Remember which routers worked this time, so that we don't upload the
+     * descriptor to them again. */
+    if (!renddesc->successful_uploads)
+      renddesc->successful_uploads = smartlist_new();
+    SMARTLIST_FOREACH(successful_uploads, const char *, c, {
+      if (!smartlist_contains_digest(renddesc->successful_uploads, c)) {
+        char *hsdir_id = tor_memdup(c, DIGEST_LEN);
+        smartlist_add(renddesc->successful_uploads, hsdir_id);
+      }
+    });
+  }
+ done:
+  smartlist_free(responsible_dirs);
+  smartlist_free(successful_uploads);
+}
+
+/** Encode and sign an up-to-date service descriptor for <b>service</b>,
+ * and upload it/them to the responsible hidden service directories.
+ */
+static void
+upload_service_descriptor(rend_service_t *service)
+{
+  time_t now = time(NULL);
+  int rendpostperiod;
+  char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  int uploaded = 0;
+
+  rendpostperiod = get_options()->RendPostPeriod;
+
+  networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && smartlist_len(c->routerstatus_list) > 0) {
+    int seconds_valid, i, j, num_descs;
+    smartlist_t *descs = smartlist_new();
+    smartlist_t *client_cookies = smartlist_new();
+    /* Either upload a single descriptor (including replicas) or one
+     * descriptor for each authorized client in case of authorization
+     * type 'stealth'. */
+    num_descs = service->auth_type == REND_STEALTH_AUTH ?
+                    smartlist_len(service->clients) : 1;
+    for (j = 0; j < num_descs; j++) {
+      crypto_pk_t *client_key = NULL;
+      rend_authorized_client_t *client = NULL;
+      smartlist_clear(client_cookies);
+      switch (service->auth_type) {
+        case REND_NO_AUTH:
+          /* Do nothing here. */
+          break;
+        case REND_BASIC_AUTH:
+          SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *,
+              cl, smartlist_add(client_cookies, cl->descriptor_cookie));
+          break;
+        case REND_STEALTH_AUTH:
+          client = smartlist_get(service->clients, j);
+          client_key = client->client_key;
+          smartlist_add(client_cookies, client->descriptor_cookie);
+          break;
+      }
+      /* Encode the current descriptor. */
+      seconds_valid = rend_encode_v2_descriptors(descs, service->desc,
+                                                 now, 0,
+                                                 service->auth_type,
+                                                 client_key,
+                                                 client_cookies);
+      if (seconds_valid < 0) {
+        log_warn(LD_BUG, "Internal error: couldn't encode service "
+                 "descriptor; not uploading.");
+        smartlist_free(descs);
+        smartlist_free(client_cookies);
+        return;
+      }
+      rend_get_service_id(service->desc->pk, serviceid);
+      if (get_options()->PublishHidServDescriptors) {
+        /* Post the current descriptors to the hidden service directories. */
+        log_info(LD_REND, "Launching upload for hidden service %s",
+                     serviceid);
+        directory_post_to_hs_dir(service->desc, descs, NULL, serviceid,
+                                 seconds_valid);
+      }
+      /* Free memory for descriptors. */
+      for (i = 0; i < smartlist_len(descs); i++)
+        rend_encoded_v2_service_descriptor_free_(smartlist_get(descs, i));
+      smartlist_clear(descs);
+      /* Update next upload time. */
+      if (seconds_valid - REND_TIME_PERIOD_OVERLAPPING_V2_DESCS
+          > rendpostperiod)
+        service->next_upload_time = now + rendpostperiod;
+      else if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS)
+        service->next_upload_time = now + seconds_valid + 1;
+      else
+        service->next_upload_time = now + seconds_valid -
+            REND_TIME_PERIOD_OVERLAPPING_V2_DESCS + 1;
+      /* Post also the next descriptors, if necessary. */
+      if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS) {
+        seconds_valid = rend_encode_v2_descriptors(descs, service->desc,
+                                                   now, 1,
+                                                   service->auth_type,
+                                                   client_key,
+                                                   client_cookies);
+        if (seconds_valid < 0) {
+          log_warn(LD_BUG, "Internal error: couldn't encode service "
+                   "descriptor; not uploading.");
+          smartlist_free(descs);
+          smartlist_free(client_cookies);
+          return;
+        }
+        if (get_options()->PublishHidServDescriptors) {
+          directory_post_to_hs_dir(service->desc, descs, NULL, serviceid,
+                                   seconds_valid);
+        }
+        /* Free memory for descriptors. */
+        for (i = 0; i < smartlist_len(descs); i++)
+          rend_encoded_v2_service_descriptor_free_(smartlist_get(descs, i));
+        smartlist_clear(descs);
+      }
+    }
+    smartlist_free(descs);
+    smartlist_free(client_cookies);
+    uploaded = 1;
+    if (get_options()->PublishHidServDescriptors) {
+      log_info(LD_REND, "Successfully uploaded v2 rend descriptors!");
+    } else {
+      log_info(LD_REND, "Successfully stored created v2 rend descriptors!");
+    }
+  }
+
+  /* If not uploaded, try again in one minute. */
+  if (!uploaded)
+    service->next_upload_time = now + 60;
+
+  /* Unmark dirty flag of this service. */
+  service->desc_is_dirty = 0;
+}
+
+/** Return the number of INTRODUCE2 cells this hidden service has received
+ * from this intro point. */
+static int
+intro_point_accepted_intro_count(rend_intro_point_t *intro)
+{
+  return intro->accepted_introduce2_count;
+}
+
+/** Return non-zero iff <b>intro</b> should 'expire' now (i.e. we
+ * should stop publishing it in new descriptors and eventually close
+ * it). */
+static int
+intro_point_should_expire_now(rend_intro_point_t *intro,
+                              time_t now)
+{
+  tor_assert(intro != NULL);
+
+  if (intro->time_published == -1) {
+    /* Don't expire an intro point if we haven't even published it yet. */
+    return 0;
+  }
+
+  if (intro_point_accepted_intro_count(intro) >=
+      intro->max_introductions) {
+    /* This intro point has been used too many times.  Expire it now. */
+    return 1;
+  }
+
+  if (intro->time_to_expire == -1) {
+    /* This intro point has been published, but we haven't picked an
+     * expiration time for it.  Pick one now. */
+    int intro_point_lifetime_seconds =
+      crypto_rand_int_range(INTRO_POINT_LIFETIME_MIN_SECONDS,
+                            INTRO_POINT_LIFETIME_MAX_SECONDS);
+
+    /* Start the expiration timer now, rather than when the intro
+     * point was first published.  There shouldn't be much of a time
+     * difference. */
+    intro->time_to_expire = now + intro_point_lifetime_seconds;
+
+    return 0;
+  }
+
+  /* This intro point has a time to expire set already.  Use it. */
+  return (now >= intro->time_to_expire);
+}
+
+/** Iterate over intro points in the given service and remove the invalid
+ * ones. For an intro point object to be considered invalid, the circuit
+ * _and_ node need to have disappeared.
+ *
+ * If the intro point should expire, it's placed into the expiring_nodes
+ * list of the service and removed from the active intro nodes list.
+ *
+ * If <b>exclude_nodes</b> is not NULL, add the valid nodes to it.
+ *
+ * If <b>retry_nodes</b> is not NULL, add the valid node to it if the
+ * circuit disappeared but the node is still in the consensus. */
+static void
+remove_invalid_intro_points(rend_service_t *service,
+                            smartlist_t *exclude_nodes,
+                            smartlist_t *retry_nodes, time_t now)
+{
+  tor_assert(service);
+
+  /* Remove any expired nodes that doesn't have a circuit. */
+  SMARTLIST_FOREACH_BEGIN(service->expiring_nodes, rend_intro_point_t *,
+                          intro) {
+    origin_circuit_t *intro_circ =
+      find_intro_circuit(intro, service->pk_digest);
+    if (intro_circ) {
+      continue;
+    }
+    /* No more circuit, cleanup the into point object. */
+    SMARTLIST_DEL_CURRENT(service->expiring_nodes, intro);
+    rend_intro_point_free(intro);
+  } SMARTLIST_FOREACH_END(intro);
+
+  SMARTLIST_FOREACH_BEGIN(service->intro_nodes, rend_intro_point_t *,
+                          intro) {
+    /* Find the introduction point node object. */
+    const node_t *node =
+      node_get_by_id(intro->extend_info->identity_digest);
+    /* Find the intro circuit, this might be NULL. */
+    origin_circuit_t *intro_circ =
+      find_intro_circuit(intro, service->pk_digest);
+
+    /* Add the valid node to the exclusion list so we don't try to establish
+     * an introduction point to it again. */
+    if (node && exclude_nodes) {
+      smartlist_add(exclude_nodes, (void*) node);
+    }
+
+    /* First, make sure we still have a valid circuit for this intro point.
+     * If we dont, we'll give up on it and make a new one. */
+    if (intro_circ == NULL) {
+      log_info(LD_REND, "Attempting to retry on %s as intro point for %s"
+               " (circuit disappeared).",
+               safe_str_client(extend_info_describe(intro->extend_info)),
+               safe_str_client(service->service_id));
+      /* We've lost the circuit for this intro point, flag it so it can be
+       * accounted for when considiring uploading a descriptor. */
+      intro->circuit_established = 0;
+
+      /* Node is gone or we've reached our maximum circuit creationg retry
+       * count, clean up everything, we'll find a new one. */
+      if (node == NULL ||
+          intro->circuit_retries >= MAX_INTRO_POINT_CIRCUIT_RETRIES) {
+        rend_intro_point_free(intro);
+        SMARTLIST_DEL_CURRENT(service->intro_nodes, intro);
+        /* We've just killed the intro point, nothing left to do. */
+        continue;
+      }
+
+      /* The intro point is still alive so let's try to use it again because
+       * we have a published descriptor containing it. Keep the intro point
+       * in the intro_nodes list because it's still valid, we are rebuilding
+       * a circuit to it. */
+      if (retry_nodes) {
+        smartlist_add(retry_nodes, intro);
+      }
+    }
+    /* else, the circuit is valid so in both cases, node being alive or not,
+     * we leave the circuit and intro point object as is. Closing the
+     * circuit here would leak new consensus timing and freeing the intro
+     * point object would make the intro circuit unusable. */
+
+    /* Now, check if intro point should expire. If it does, queue it so
+     * it can be cleaned up once it has been replaced properly. */
+    if (intro_point_should_expire_now(intro, now)) {
+      log_info(LD_REND, "Expiring %s as intro point for %s.",
+               safe_str_client(extend_info_describe(intro->extend_info)),
+               safe_str_client(service->service_id));
+      /* We might have put it in the retry list if so, undo. */
+      if (retry_nodes) {
+        smartlist_remove(retry_nodes, intro);
+      }
+      smartlist_add(service->expiring_nodes, intro);
+      SMARTLIST_DEL_CURRENT(service->intro_nodes, intro);
+      /* Intro point is expired, we need a new one thus don't consider it
+       * anymore has a valid established intro point. */
+      intro->circuit_established = 0;
+    }
+  } SMARTLIST_FOREACH_END(intro);
+}
+
+/** A new descriptor has been successfully uploaded for the given
+ * <b>rend_data</b>. Remove and free the expiring nodes from the associated
+ * service. */
+void
+rend_service_desc_has_uploaded(const rend_data_t *rend_data)
+{
+  rend_service_t *service;
+  const char *onion_address;
+
+  tor_assert(rend_data);
+
+  onion_address = rend_data_get_address(rend_data);
+
+  service = rend_service_get_by_service_id(onion_address);
+  if (service == NULL) {
+    return;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(service->expiring_nodes, rend_intro_point_t *,
+                          intro) {
+    origin_circuit_t *intro_circ =
+      find_intro_circuit(intro, service->pk_digest);
+    if (intro_circ != NULL) {
+      circuit_mark_for_close(TO_CIRCUIT(intro_circ),
+                             END_CIRC_REASON_FINISHED);
+    }
+    SMARTLIST_DEL_CURRENT(service->expiring_nodes, intro);
+    rend_intro_point_free(intro);
+  } SMARTLIST_FOREACH_END(intro);
+}
+
+/** Don't try to build more than this many circuits before giving up
+ * for a while. Dynamically calculated based on the configured number of
+ * introduction points for the service, n_intro_points_wanted. */
+static int
+rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted)
+{
+  /* Allow all but one of the initial connections to fail and be
+   * retried. (If all fail, we *want* to wait, because something is broken.) */
+  tor_assert(n_intro_points_wanted <= NUM_INTRO_POINTS_MAX);
+
+  /* For the normal use case, 3 intro points plus 2 extra for performance and
+   * allow that twice because once every 24h or so, we can do it twice for two
+   * descriptors that is the current one and the next one. So (3 + 2) * 2 ==
+   * 12 allowed attempts for one period. */
+  return ((n_intro_points_wanted + NUM_INTRO_POINTS_EXTRA) * 2);
+}
+
+/** For every service, check how many intro points it currently has, and:
+ *  - Invalidate introdution points based on specific criteria, see
+ *  remove_invalid_intro_points comments.
+ *  - Pick new intro points as necessary.
+ *  - Launch circuits to any new intro points.
+ *
+ * This is called once a second by the main loop.
+ */
+void
+rend_consider_services_intro_points(time_t now)
+{
+  int i;
+  const or_options_t *options = get_options();
+  /* Are we in single onion mode? */
+  const int allow_direct = rend_service_allow_non_anonymous_connection(
+                                                                get_options());
+  /* List of nodes we need to _exclude_ when choosing a new node to
+   * establish an intro point to. */
+  smartlist_t *exclude_nodes;
+  /* List of nodes we need to retry to build a circuit on them because the
+   * node is valid but circuit died. */
+  smartlist_t *retry_nodes;
+
+  if (!have_completed_a_circuit())
+    return;
+
+  exclude_nodes = smartlist_new();
+  retry_nodes = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, service) {
+    int r;
+    /* Number of intro points we want to open and add to the intro nodes
+     * list of the service. */
+    unsigned int n_intro_points_to_open;
+    /* Have an unsigned len so we can use it to compare values else gcc is
+     * not happy with unmatching signed comparaison. */
+    unsigned int intro_nodes_len;
+    /* Different service are allowed to have the same introduction point as
+     * long as they are on different circuit thus why we clear this list. */
+    smartlist_clear(exclude_nodes);
+    smartlist_clear(retry_nodes);
+
+    /* Cleanup the invalid intro points and save the node objects, if any,
+     * in the exclude_nodes and retry_nodes lists. */
+    remove_invalid_intro_points(service, exclude_nodes, retry_nodes, now);
+
+    /* This retry period is important here so we don't stress circuit
+     * creation. */
+
+    if (now > service->intro_period_started + INTRO_CIRC_RETRY_PERIOD) {
+      /* One period has elapsed:
+       *  - if we stopped, we can try building circuits again,
+       *  - if we haven't, we reset the circuit creation counts. */
+      rend_log_intro_limit(service, LOG_INFO);
+      service->intro_period_started = now;
+      service->n_intro_circuits_launched = 0;
+    } else if (service->n_intro_circuits_launched >=
+               rend_max_intro_circs_per_period(
+                                      service->n_intro_points_wanted)) {
+      /* We have failed too many times in this period; wait for the next
+       * one before we try to initiate any more connections. */
+      rend_log_intro_limit(service, LOG_WARN);
+      continue;
+    }
+
+    /* Let's try to rebuild circuit on the nodes we want to retry on. */
+    SMARTLIST_FOREACH_BEGIN(retry_nodes, rend_intro_point_t *, intro) {
+      r = rend_service_launch_establish_intro(service, intro);
+      if (r < 0) {
+        log_warn(LD_REND, "Error launching circuit to node %s for service %s.",
+                 safe_str_client(extend_info_describe(intro->extend_info)),
+                 safe_str_client(service->service_id));
+        /* Unable to launch a circuit to that intro point, remove it from
+         * the valid list so we can create a new one. */
+        smartlist_remove(service->intro_nodes, intro);
+        rend_intro_point_free(intro);
+        continue;
+      }
+      intro->circuit_retries++;
+    } SMARTLIST_FOREACH_END(intro);
+
+    /* Avoid mismatched signed comparaison below. */
+    intro_nodes_len = (unsigned int) smartlist_len(service->intro_nodes);
+
+    /* Quiescent state, we have more or the equal amount of wanted node for
+     * this service. Proceed to the next service. We can have more nodes
+     * because we launch extra preemptive circuits if our intro nodes list was
+     * originally empty for performance reasons. */
+    if (intro_nodes_len >= service->n_intro_points_wanted) {
+      continue;
+    }
+
+    /* Number of intro points we want to open which is the wanted amount minus
+     * the current amount of valid nodes. We know that this won't underflow
+     * because of the check above. */
+    n_intro_points_to_open = service->n_intro_points_wanted - intro_nodes_len;
+    if (intro_nodes_len == 0) {
+      /* We want to end up with n_intro_points_wanted intro points, but if
+       * we have no intro points at all (chances are they all cycled or we
+       * are starting up), we launch NUM_INTRO_POINTS_EXTRA extra circuits
+       * and use the first n_intro_points_wanted that complete. See proposal
+       * #155, section 4 for the rationale of this which is purely for
+       * performance.
+       *
+       * The ones after the first n_intro_points_to_open will be converted
+       * to 'general' internal circuits in rend_service_intro_has_opened(),
+       * and then we'll drop them from the list of intro points. */
+      n_intro_points_to_open += NUM_INTRO_POINTS_EXTRA;
+    }
+
+    for (i = 0; i < (int) n_intro_points_to_open; i++) {
+      const node_t *node;
+      rend_intro_point_t *intro;
+      router_crn_flags_t flags = CRN_NEED_UPTIME|CRN_NEED_DESC;
+      router_crn_flags_t direct_flags = flags;
+      direct_flags |= CRN_PREF_ADDR;
+      direct_flags |= CRN_DIRECT_CONN;
+
+      node = router_choose_random_node(exclude_nodes,
+                                       options->ExcludeNodes,
+                                       allow_direct ? direct_flags : flags);
+      /* If we are in single onion mode, retry node selection for a 3-hop
+       * path */
+      if (allow_direct && !node) {
+        log_info(LD_REND,
+                 "Unable to find an intro point that we can connect to "
+                 "directly for %s, falling back to a 3-hop path.",
+                 safe_str_client(service->service_id));
+        node = router_choose_random_node(exclude_nodes,
+                                         options->ExcludeNodes, flags);
+      }
+
+      if (!node) {
+        log_warn(LD_REND,
+                 "We only have %d introduction points established for %s; "
+                 "wanted %u.",
+                 smartlist_len(service->intro_nodes),
+                 safe_str_client(service->service_id),
+                 n_intro_points_to_open);
+        break;
+      }
+      /* Add the chosen node to the exclusion list in order to avoid picking
+       * it again in the next iteration. */
+      smartlist_add(exclude_nodes, (void*)node);
+      intro = tor_malloc_zero(sizeof(rend_intro_point_t));
+      /* extend_info is for clients, so we want the multi-hop primary ORPort,
+       * even if we are a single onion service and intend to connect to it
+       * directly ourselves. */
+      intro->extend_info = extend_info_from_node(node, 0);
+      if (BUG(intro->extend_info == NULL)) {
+        break;
+      }
+      intro->intro_key = crypto_pk_new();
+      const int fail = crypto_pk_generate_key(intro->intro_key);
+      tor_assert(!fail);
+      intro->time_published = -1;
+      intro->time_to_expire = -1;
+      intro->max_introductions =
+        crypto_rand_int_range(INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS,
+                              INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS);
+      smartlist_add(service->intro_nodes, intro);
+      log_info(LD_REND, "Picked router %s as an intro point for %s.",
+               safe_str_client(node_describe(node)),
+               safe_str_client(service->service_id));
+      /* Establish new introduction circuit to our chosen intro point. */
+      r = rend_service_launch_establish_intro(service, intro);
+      if (r < 0) {
+        log_warn(LD_REND, "Error launching circuit to node %s for service %s.",
+                 safe_str_client(extend_info_describe(intro->extend_info)),
+                 safe_str_client(service->service_id));
+        /* This funcion will be called again by the main loop so this intro
+         * point without a intro circuit will be retried on or removed after
+         * a maximum number of attempts. */
+      }
+    }
+  } SMARTLIST_FOREACH_END(service);
+  smartlist_free(exclude_nodes);
+  smartlist_free(retry_nodes);
+}
+
+#define MIN_REND_INITIAL_POST_DELAY (30)
+#define MIN_REND_INITIAL_POST_DELAY_TESTING (5)
+
+/** Regenerate and upload rendezvous service descriptors for all
+ * services, if necessary. If the descriptor has been dirty enough
+ * for long enough, definitely upload; else only upload when the
+ * periodic timeout has expired.
+ *
+ * For the first upload, pick a random time between now and two periods
+ * from now, and pick it independently for each service.
+ */
+void
+rend_consider_services_upload(time_t now)
+{
+  int i;
+  rend_service_t *service;
+  const or_options_t *options = get_options();
+  int rendpostperiod = options->RendPostPeriod;
+  int rendinitialpostdelay = (options->TestingTorNetwork ?
+                              MIN_REND_INITIAL_POST_DELAY_TESTING :
+                              MIN_REND_INITIAL_POST_DELAY);
+
+  for (i=0; i < smartlist_len(rend_service_list); ++i) {
+    service = smartlist_get(rend_service_list, i);
+    if (!service->next_upload_time) { /* never been uploaded yet */
+      /* The fixed lower bound of rendinitialpostdelay seconds ensures that
+       * the descriptor is stable before being published. See comment below. */
+      service->next_upload_time =
+        now + rendinitialpostdelay + crypto_rand_int(2*rendpostperiod);
+      /* Single Onion Services prioritise availability over hiding their
+       * startup time, as their IP address is publicly discoverable anyway.
+       */
+      if (rend_service_reveal_startup_time(options)) {
+        service->next_upload_time = now + rendinitialpostdelay;
+      }
+    }
+    /* Does every introduction points have been established? */
+    unsigned int intro_points_ready =
+      count_established_intro_points(service) >=
+        service->n_intro_points_wanted;
+    if (intro_points_ready &&
+        (service->next_upload_time < now ||
+        (service->desc_is_dirty &&
+         service->desc_is_dirty < now-rendinitialpostdelay))) {
+      /* if it's time, or if the directory servers have a wrong service
+       * descriptor and ours has been stable for rendinitialpostdelay seconds,
+       * upload a new one of each format. */
+      rend_service_update_descriptor(service);
+      upload_service_descriptor(service);
+    }
+  }
+}
+
+/** True if the list of available router descriptors might have changed so
+ * that we should have a look whether we can republish previously failed
+ * rendezvous service descriptors. */
+static int consider_republishing_rend_descriptors = 1;
+
+/** Called when our internal view of the directory has changed, so that we
+ * might have router descriptors of hidden service directories available that
+ * we did not have before. */
+void
+rend_hsdir_routers_changed(void)
+{
+  consider_republishing_rend_descriptors = 1;
+}
+
+/** Consider republication of v2 rendezvous service descriptors that failed
+ * previously, but without regenerating descriptor contents.
+ */
+void
+rend_consider_descriptor_republication(void)
+{
+  int i;
+  rend_service_t *service;
+
+  if (!consider_republishing_rend_descriptors)
+    return;
+  consider_republishing_rend_descriptors = 0;
+
+  if (!get_options()->PublishHidServDescriptors)
+    return;
+
+  for (i=0; i < smartlist_len(rend_service_list); ++i) {
+    service = smartlist_get(rend_service_list, i);
+    if (service->desc && !service->desc->all_uploads_performed) {
+      /* If we failed in uploading a descriptor last time, try again *without*
+       * updating the descriptor's contents. */
+      upload_service_descriptor(service);
+    }
+  }
+}
+
+/** Log the status of introduction points for all rendezvous services
+ * at log severity <b>severity</b>.
+ */
+void
+rend_service_dump_stats(int severity)
+{
+  int i,j;
+  rend_service_t *service;
+  rend_intro_point_t *intro;
+  const char *safe_name;
+  origin_circuit_t *circ;
+
+  for (i=0; i < smartlist_len(rend_service_list); ++i) {
+    service = smartlist_get(rend_service_list, i);
+    tor_log(severity, LD_GENERAL, "Service configured in %s:",
+            rend_service_escaped_dir(service));
+    for (j=0; j < smartlist_len(service->intro_nodes); ++j) {
+      intro = smartlist_get(service->intro_nodes, j);
+      safe_name = safe_str_client(intro->extend_info->nickname);
+
+      circ = find_intro_circuit(intro, service->pk_digest);
+      if (!circ) {
+        tor_log(severity, LD_GENERAL, "  Intro point %d at %s: no circuit",
+            j, safe_name);
+        continue;
+      }
+      tor_log(severity, LD_GENERAL, "  Intro point %d at %s: circuit is %s",
+          j, safe_name, circuit_state_to_string(circ->base_.state));
+    }
+  }
+}
+
+/** Given <b>conn</b>, a rendezvous exit stream, look up the hidden service for
+ * <b>circ</b>, and look up the port and address based on conn-\>port.
+ * Assign the actual conn-\>addr and conn-\>port. Return -2 on failure
+ * for which the circuit should be closed, -1 on other failure,
+ * or 0 for success.
+ */
+int
+rend_service_set_connection_addr_port(edge_connection_t *conn,
+                                      origin_circuit_t *circ)
+{
+  rend_service_t *service;
+  char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  const char *rend_pk_digest;
+
+  tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
+  tor_assert(circ->rend_data);
+  log_debug(LD_REND,"beginning to hunt for addr/port");
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circ->rend_data, NULL);
+  base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                rend_pk_digest, REND_SERVICE_ID_LEN);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
+  if (!service) {
+    log_warn(LD_REND, "Couldn't find any service associated with pk %s on "
+             "rendezvous circuit %u; closing.",
+             serviceid, (unsigned)circ->base_.n_circ_id);
+    return -2;
+  }
+  if (service->max_streams_per_circuit > 0) {
+    /* Enforce the streams-per-circuit limit, and refuse to provide a
+     * mapping if this circuit will exceed the limit. */
+#define MAX_STREAM_WARN_INTERVAL 600
+    static struct ratelim_t stream_ratelim =
+        RATELIM_INIT(MAX_STREAM_WARN_INTERVAL);
+    if (circ->rend_data->nr_streams >= service->max_streams_per_circuit) {
+      log_fn_ratelim(&stream_ratelim, LOG_WARN, LD_REND,
+                     "Maximum streams per circuit limit reached on rendezvous "
+                     "circuit %u; %s.  Circuit has %d out of %d streams.",
+                     (unsigned)circ->base_.n_circ_id,
+                     service->max_streams_close_circuit ?
+                       "closing circuit" :
+                       "ignoring open stream request",
+                     circ->rend_data->nr_streams,
+                     service->max_streams_per_circuit);
+      return service->max_streams_close_circuit ? -2 : -1;
+    }
+  }
+
+  if (hs_set_conn_addr_port(service->ports, conn) == 0) {
+    /* Successfully set the port to the connection. We are done. */
+    return 0;
+  }
+
+  log_info(LD_REND,
+           "No virtual port mapping exists for port %d on service %s",
+           conn->base_.port, serviceid);
+
+  if (service->allow_unknown_ports)
+    return -1;
+  else
+    return -2;
+}
+
+/* Are HiddenServiceSingleHopMode and HiddenServiceNonAnonymousMode consistent?
+ */
+static int
+rend_service_non_anonymous_mode_consistent(const or_options_t *options)
+{
+  /* !! is used to make these options boolean */
+  return (!! options->HiddenServiceSingleHopMode ==
+          !! options->HiddenServiceNonAnonymousMode);
+}
+
+/* Do the options allow onion services to make direct (non-anonymous)
+ * connections to introduction or rendezvous points?
+ * Must only be called after options_validate_single_onion() has successfully
+ * checked onion service option consistency.
+ * Returns true if tor is in HiddenServiceSingleHopMode. */
+int
+rend_service_allow_non_anonymous_connection(const or_options_t *options)
+{
+  tor_assert(rend_service_non_anonymous_mode_consistent(options));
+  return options->HiddenServiceSingleHopMode ? 1 : 0;
+}
+
+/* Do the options allow us to reveal the exact startup time of the onion
+ * service?
+ * Single Onion Services prioritise availability over hiding their
+ * startup time, as their IP address is publicly discoverable anyway.
+ * Must only be called after options_validate_single_onion() has successfully
+ * checked onion service option consistency.
+ * Returns true if tor is in non-anonymous hidden service mode. */
+int
+rend_service_reveal_startup_time(const or_options_t *options)
+{
+  tor_assert(rend_service_non_anonymous_mode_consistent(options));
+  return rend_service_non_anonymous_mode_enabled(options);
+}
+
+/* Is non-anonymous mode enabled using the HiddenServiceNonAnonymousMode
+ * config option?
+ * Must only be called after options_validate_single_onion() has successfully
+ * checked onion service option consistency.
+ */
+int
+rend_service_non_anonymous_mode_enabled(const or_options_t *options)
+{
+  tor_assert(rend_service_non_anonymous_mode_consistent(options));
+  return options->HiddenServiceNonAnonymousMode ? 1 : 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC void
+set_rend_service_list(smartlist_t *new_list)
+{
+  rend_service_list = new_list;
+}
+
+STATIC void
+set_rend_rend_service_staging_list(smartlist_t *new_list)
+{
+  rend_service_staging_list = new_list;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/rend/rendservice.h b/src/feature/rend/rendservice.h
new file mode 100644
index 0000000000..35962df7f3
--- /dev/null
+++ b/src/feature/rend/rendservice.h
@@ -0,0 +1,221 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendservice.h
+ * \brief Header file for rendservice.c.
+ **/
+
+#ifndef TOR_RENDSERVICE_H
+#define TOR_RENDSERVICE_H
+
+#include "or/or.h"
+#include "or/hs_service.h"
+
+typedef struct rend_intro_cell_t rend_intro_cell_t;
+struct config_line_t;
+
+/* This can be used for both INTRODUCE1 and INTRODUCE2 */
+
+struct rend_intro_cell_t {
+  /* Is this an INTRODUCE1 or INTRODUCE2? (set to 1 or 2) */
+  uint8_t type;
+  /* Public key digest */
+  uint8_t pk[DIGEST_LEN];
+  /* Optionally, store ciphertext here */
+  uint8_t *ciphertext;
+  ssize_t ciphertext_len;
+  /* Optionally, store plaintext */
+  uint8_t *plaintext;
+  ssize_t plaintext_len;
+  /* Have we parsed the plaintext? */
+  uint8_t parsed;
+  /* intro protocol version (0, 1, 2 or 3) */
+  uint8_t version;
+  /* Version-specific parts */
+  union {
+    struct {
+      /* Rendezvous point nickname or hex-encoded key digest */
+      uint8_t rp[42];
+    } v0_v1;
+    struct {
+      /* The extend_info_t struct has everything v2 uses */
+      extend_info_t *extend_info;
+    } v2;
+    struct {
+      /* Auth type used */
+      uint8_t auth_type;
+      /* Length of auth data */
+      uint16_t auth_len;
+      /* Auth data */
+      uint8_t *auth_data;
+      /* Rendezvous point's IP address/port, identity digest and onion key */
+      extend_info_t *extend_info;
+    } v3;
+  } u;
+  /* Rendezvous cookie */
+  uint8_t rc[REND_COOKIE_LEN];
+  /* Diffie-Hellman data */
+  uint8_t dh[DH1024_KEY_LEN];
+};
+
+#ifdef RENDSERVICE_PRIVATE
+
+/** Represents a single hidden service running at this OP. */
+typedef struct rend_service_t {
+  /* Fields specified in config file */
+  char *directory; /**< where in the filesystem it stores it. Will be NULL if
+                    * this service is ephemeral. */
+  int dir_group_readable; /**< if 1, allow group read
+                             permissions on directory */
+  smartlist_t *ports; /**< List of rend_service_port_config_t */
+  rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client
+                               * authorization is performed. */
+  smartlist_t *clients; /**< List of rend_authorized_client_t's of
+                         * clients that may access our service. Can be NULL
+                         * if no client authorization is performed. */
+  /* Other fields */
+  crypto_pk_t *private_key; /**< Permanent hidden-service key. */
+  char service_id[REND_SERVICE_ID_LEN_BASE32+1]; /**< Onion address without
+                                                  * '.onion' */
+  char pk_digest[DIGEST_LEN]; /**< Hash of permanent hidden-service key. */
+  smartlist_t *intro_nodes; /**< List of rend_intro_point_t's we have,
+                             * or are trying to establish. */
+  /** List of rend_intro_point_t that are expiring. They are removed once
+   * the new descriptor is successfully uploaded. A node in this list CAN
+   * NOT appear in the intro_nodes list. */
+  smartlist_t *expiring_nodes;
+  time_t intro_period_started; /**< Start of the current period to build
+                                * introduction points. */
+  int n_intro_circuits_launched; /**< Count of intro circuits we have
+                                  * established in this period. */
+  unsigned int n_intro_points_wanted; /**< Number of intro points this
+                                       * service wants to have open. */
+  rend_service_descriptor_t *desc; /**< Current hidden service descriptor. */
+  time_t desc_is_dirty; /**< Time at which changes to the hidden service
+                         * descriptor content occurred, or 0 if it's
+                         * up-to-date. */
+  time_t next_upload_time; /**< Scheduled next hidden service descriptor
+                            * upload time. */
+  /** Replay cache for Diffie-Hellman values of INTRODUCE2 cells, to
+   * detect repeats.  Clients may send INTRODUCE1 cells for the same
+   * rendezvous point through two or more different introduction points;
+   * when they do, this keeps us from launching multiple simultaneous attempts
+   * to connect to the same rend point. */
+  replaycache_t *accepted_intro_dh_parts;
+  /** If true, we don't close circuits for making requests to unsupported
+   * ports. */
+  int allow_unknown_ports;
+  /** The maximum number of simultaneous streams-per-circuit that are allowed
+   * to be established, or 0 if no limit is set.
+   */
+  int max_streams_per_circuit;
+  /** If true, we close circuits that exceed the max_streams_per_circuit
+   * limit.  */
+  int max_streams_close_circuit;
+} rend_service_t;
+
+STATIC void rend_service_free_(rend_service_t *service);
+#define rend_service_free(s) \
+  FREE_AND_NULL(rend_service_t, rend_service_free_, (s))
+STATIC char *rend_service_sos_poison_path(const rend_service_t *service);
+STATIC int rend_service_verify_single_onion_poison(
+                                                  const rend_service_t *s,
+                                                  const or_options_t *options);
+STATIC int rend_service_poison_new_single_onion_dir(
+                                                  const rend_service_t *s,
+                                                  const or_options_t* options);
+#ifdef TOR_UNIT_TESTS
+
+STATIC void set_rend_service_list(smartlist_t *new_list);
+STATIC void set_rend_rend_service_staging_list(smartlist_t *new_list);
+STATIC void rend_service_prune_list_impl_(void);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(RENDSERVICE_PRIVATE) */
+
+int rend_num_services(void);
+int rend_config_service(const struct config_line_t *line_,
+                        const or_options_t *options,
+                        hs_service_config_t *config);
+void rend_service_prune_list(void);
+void rend_service_free_staging_list(void);
+int rend_service_load_all_keys(const smartlist_t *service_list);
+void rend_services_add_filenames_to_lists(smartlist_t *open_lst,
+                                          smartlist_t *stat_lst);
+void rend_consider_services_intro_points(time_t now);
+void rend_consider_services_upload(time_t now);
+void rend_hsdir_routers_changed(void);
+void rend_consider_descriptor_republication(void);
+
+void rend_service_intro_has_opened(origin_circuit_t *circuit);
+int rend_service_intro_established(origin_circuit_t *circuit,
+                                   const uint8_t *request,
+                                   size_t request_len);
+void rend_service_rendezvous_has_opened(origin_circuit_t *circuit);
+int rend_service_receive_introduction(origin_circuit_t *circuit,
+                                      const uint8_t *request,
+                                      size_t request_len);
+int rend_service_decrypt_intro(rend_intro_cell_t *request,
+                               crypto_pk_t *key,
+                               char **err_msg_out);
+void rend_service_free_intro_(rend_intro_cell_t *request);
+#define rend_service_free_intro(req) do {       \
+    rend_service_free_intro_(req);              \
+    (req) = NULL;                               \
+  } while (0)
+rend_intro_cell_t * rend_service_begin_parse_intro(const uint8_t *request,
+                                                   size_t request_len,
+                                                   uint8_t type,
+                                                   char **err_msg_out);
+int rend_service_parse_intro_plaintext(rend_intro_cell_t *intro,
+                                       char **err_msg_out);
+ssize_t rend_service_encode_establish_intro_cell(char *cell_body_out,
+                                                 size_t cell_body_out_len,
+                                                 crypto_pk_t *intro_key,
+                                                 const char *rend_circ_nonce);
+int rend_service_validate_intro_late(const rend_intro_cell_t *intro,
+                                     char **err_msg_out);
+void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc);
+int rend_service_set_connection_addr_port(edge_connection_t *conn,
+                                          origin_circuit_t *circ);
+void rend_service_dump_stats(int severity);
+void rend_service_free_all(void);
+void rend_service_init(void);
+
+rend_service_port_config_t *rend_service_parse_port_config(const char *string,
+                                                           const char *sep,
+                                                           char **err_msg_out);
+void rend_service_port_config_free_(rend_service_port_config_t *p);
+#define rend_service_port_config_free(p) \
+  FREE_AND_NULL(rend_service_port_config_t, rend_service_port_config_free_, \
+                (p))
+
+void rend_authorized_client_free_(rend_authorized_client_t *client);
+#define rend_authorized_client_free(client) \
+  FREE_AND_NULL(rend_authorized_client_t, rend_authorized_client_free_, \
+                (client))
+
+hs_service_add_ephemeral_status_t rend_service_add_ephemeral(crypto_pk_t *pk,
+                               smartlist_t *ports,
+                               int max_streams_per_circuit,
+                               int max_streams_close_circuit,
+                               rend_auth_type_t auth_type,
+                               smartlist_t *auth_clients,
+                               char **service_id_out);
+int rend_service_del_ephemeral(const char *service_id);
+
+void directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
+                              smartlist_t *descs, smartlist_t *hs_dirs,
+                              const char *service_id, int seconds_valid);
+void rend_service_desc_has_uploaded(const rend_data_t *rend_data);
+
+int rend_service_allow_non_anonymous_connection(const or_options_t *options);
+int rend_service_reveal_startup_time(const or_options_t *options);
+int rend_service_non_anonymous_mode_enabled(const or_options_t *options);
+
+#endif /* !defined(TOR_RENDSERVICE_H) */
diff --git a/src/feature/stats/geoip.c b/src/feature/stats/geoip.c
new file mode 100644
index 0000000000..3e6f20ea3f
--- /dev/null
+++ b/src/feature/stats/geoip.c
@@ -0,0 +1,1889 @@
+/* Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file geoip.c
+ * \brief Functions related to maintaining an IP-to-country database;
+ * to summarizing client connections by country to entry guards, bridges,
+ * and directory servers; and for statistics on answering network status
+ * requests.
+ *
+ * There are two main kinds of functions in this module: geoip functions,
+ * which map groups of IPv4 and IPv6 addresses to country codes, and
+ * statistical functions, which collect statistics about different kinds of
+ * per-country usage.
+ *
+ * The geoip lookup tables are implemented as sorted lists of disjoint address
+ * ranges, each mapping to a singleton geoip_country_t.  These country objects
+ * are also indexed by their names in a hashtable.
+ *
+ * The tables are populated from disk at startup by the geoip_load_file()
+ * function.  For more information on the file format they read, see that
+ * function.  See the scripts and the README file in src/config for more
+ * information about how those files are generated.
+ *
+ * Tor uses GeoIP information in order to implement user requests (such as
+ * ExcludeNodes {cc}), and to keep track of how much usage relays are getting
+ * for each country.
+ */
+
+#define GEOIP_PRIVATE
+#include "or/or.h"
+#include "ht.h"
+#include "lib/container/buffers.h"
+#include "or/config.h"
+#include "or/control.h"
+#include "or/dnsserv.h"
+#include "or/dos.h"
+#include "or/geoip.h"
+#include "or/routerlist.h"
+
+#include "lib/container/order.h"
+#include "lib/time/tvdiff.h"
+
+static void init_geoip_countries(void);
+
+/** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */
+typedef struct geoip_ipv4_entry_t {
+  uint32_t ip_low; /**< The lowest IP in the range, in host order */
+  uint32_t ip_high; /**< The highest IP in the range, in host order */
+  intptr_t country; /**< An index into geoip_countries */
+} geoip_ipv4_entry_t;
+
+/** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */
+typedef struct geoip_ipv6_entry_t {
+  struct in6_addr ip_low; /**< The lowest IP in the range, in host order */
+  struct in6_addr ip_high; /**< The highest IP in the range, in host order */
+  intptr_t country; /**< An index into geoip_countries */
+} geoip_ipv6_entry_t;
+
+/** A per-country record for GeoIP request history. */
+typedef struct geoip_country_t {
+  char countrycode[3];
+  uint32_t n_v3_ns_requests;
+} geoip_country_t;
+
+/** A list of geoip_country_t */
+static smartlist_t *geoip_countries = NULL;
+/** A map from lowercased country codes to their position in geoip_countries.
+ * The index is encoded in the pointer, and 1 is added so that NULL can mean
+ * not found. */
+static strmap_t *country_idxplus1_by_lc_code = NULL;
+/** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted
+ * by their respective ip_low. */
+static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
+
+/** SHA1 digest of the GeoIP files to include in extra-info descriptors. */
+static char geoip_digest[DIGEST_LEN];
+static char geoip6_digest[DIGEST_LEN];
+
+/* Total size in bytes of the geoip client history cache. Used by the OOM
+ * handler. */
+static size_t geoip_client_history_cache_size;
+
+/* Increment the geoip client history cache size counter with the given bytes.
+ * This prevents an overflow and set it to its maximum in that case. */
+static inline void
+geoip_increment_client_history_cache_size(size_t bytes)
+{
+  /* This is shockingly high, lets log it so it can be reported. */
+  IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) {
+    geoip_client_history_cache_size = SIZE_MAX;
+    return;
+  }
+  geoip_client_history_cache_size += bytes;
+}
+
+/* Decrement the geoip client history cache size counter with the given bytes.
+ * This prevents an underflow and set it to 0 in that case. */
+static inline void
+geoip_decrement_client_history_cache_size(size_t bytes)
+{
+  /* Going below 0 means that we either allocated an entry without
+   * incrementing the counter or we have different sizes when allocating and
+   * freeing. It shouldn't happened so log it. */
+  IF_BUG_ONCE(geoip_client_history_cache_size < bytes) {
+    geoip_client_history_cache_size = 0;
+    return;
+  }
+  geoip_client_history_cache_size -= bytes;
+}
+
+/** Return the index of the <b>country</b>'s entry in the GeoIP
+ * country list if it is a valid 2-letter country code, otherwise
+ * return -1. */
+MOCK_IMPL(country_t,
+geoip_get_country,(const char *country))
+{
+  void *idxplus1_;
+  intptr_t idx;
+
+  idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
+  if (!idxplus1_)
+    return -1;
+
+  idx = ((uintptr_t)idxplus1_)-1;
+  return (country_t)idx;
+}
+
+/** Add an entry to a GeoIP table, mapping all IP addresses between <b>low</b>
+ * and <b>high</b>, inclusive, to the 2-letter country code <b>country</b>. */
+static void
+geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high,
+                const char *country)
+{
+  intptr_t idx;
+  void *idxplus1_;
+
+  IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high))
+    return;
+  IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0)
+    return;
+
+  idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
+
+  if (!idxplus1_) {
+    geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t));
+    strlcpy(c->countrycode, country, sizeof(c->countrycode));
+    tor_strlower(c->countrycode);
+    smartlist_add(geoip_countries, c);
+    idx = smartlist_len(geoip_countries) - 1;
+    strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1));
+  } else {
+    idx = ((uintptr_t)idxplus1_)-1;
+  }
+  {
+    geoip_country_t *c = smartlist_get(geoip_countries, (int)idx);
+    tor_assert(!strcasecmp(c->countrycode, country));
+  }
+
+  if (tor_addr_family(low) == AF_INET) {
+    geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t));
+    ent->ip_low = tor_addr_to_ipv4h(low);
+    ent->ip_high = tor_addr_to_ipv4h(high);
+    ent->country = idx;
+    smartlist_add(geoip_ipv4_entries, ent);
+  } else if (tor_addr_family(low) == AF_INET6) {
+    geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t));
+    ent->ip_low = *tor_addr_to_in6_assert(low);
+    ent->ip_high = *tor_addr_to_in6_assert(high);
+    ent->country = idx;
+    smartlist_add(geoip_ipv6_entries, ent);
+  }
+}
+
+/** Add an entry to the GeoIP table indicated by <b>family</b>,
+ * parsing it from <b>line</b>. The format is as for geoip_load_file(). */
+STATIC int
+geoip_parse_entry(const char *line, sa_family_t family)
+{
+  tor_addr_t low_addr, high_addr;
+  char c[3];
+  char *country = NULL;
+
+  if (!geoip_countries)
+    init_geoip_countries();
+  if (family == AF_INET) {
+    if (!geoip_ipv4_entries)
+      geoip_ipv4_entries = smartlist_new();
+  } else if (family == AF_INET6) {
+    if (!geoip_ipv6_entries)
+      geoip_ipv6_entries = smartlist_new();
+  } else {
+    log_warn(LD_GENERAL, "Unsupported family: %d", family);
+    return -1;
+  }
+
+  while (TOR_ISSPACE(*line))
+    ++line;
+  if (*line == '#')
+    return 0;
+
+  char buf[512];
+  if (family == AF_INET) {
+    unsigned int low, high;
+    if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 ||
+        tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) {
+      tor_addr_from_ipv4h(&low_addr, low);
+      tor_addr_from_ipv4h(&high_addr, high);
+    } else
+      goto fail;
+    country = c;
+  } else {                      /* AF_INET6 */
+    char *low_str, *high_str;
+    struct in6_addr low, high;
+    char *strtok_state;
+    strlcpy(buf, line, sizeof(buf));
+    low_str = tor_strtok_r(buf, ",", &strtok_state);
+    if (!low_str)
+      goto fail;
+    high_str = tor_strtok_r(NULL, ",", &strtok_state);
+    if (!high_str)
+      goto fail;
+    country = tor_strtok_r(NULL, "\n", &strtok_state);
+    if (!country)
+      goto fail;
+    if (strlen(country) != 2)
+      goto fail;
+    if (tor_inet_pton(AF_INET6, low_str, &low) <= 0)
+      goto fail;
+    tor_addr_from_in6(&low_addr, &low);
+    if (tor_inet_pton(AF_INET6, high_str, &high) <= 0)
+      goto fail;
+    tor_addr_from_in6(&high_addr, &high);
+  }
+  geoip_add_entry(&low_addr, &high_addr, country);
+  return 0;
+
+  fail:
+  log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s",
+           family == AF_INET ? "IPv4" : "IPv6", escaped(line));
+  return -1;
+}
+
+/** Sorting helper: return -1, 1, or 0 based on comparison of two
+ * geoip_ipv4_entry_t */
+static int
+geoip_ipv4_compare_entries_(const void **_a, const void **_b)
+{
+  const geoip_ipv4_entry_t *a = *_a, *b = *_b;
+  if (a->ip_low < b->ip_low)
+    return -1;
+  else if (a->ip_low > b->ip_low)
+    return 1;
+  else
+    return 0;
+}
+
+/** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer
+ * to a uint32_t in host order) to a geoip_ipv4_entry_t */
+static int
+geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member)
+{
+  /* No alignment issue here, since _key really is a pointer to uint32_t */
+  const uint32_t addr = *(uint32_t *)_key;
+  const geoip_ipv4_entry_t *entry = *_member;
+  if (addr < entry->ip_low)
+    return -1;
+  else if (addr > entry->ip_high)
+    return 1;
+  else
+    return 0;
+}
+
+/** Sorting helper: return -1, 1, or 0 based on comparison of two
+ * geoip_ipv6_entry_t */
+static int
+geoip_ipv6_compare_entries_(const void **_a, const void **_b)
+{
+  const geoip_ipv6_entry_t *a = *_a, *b = *_b;
+  return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr,
+                     sizeof(struct in6_addr));
+}
+
+/** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6
+ * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */
+static int
+geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member)
+{
+  const struct in6_addr *addr = (struct in6_addr *)_key;
+  const geoip_ipv6_entry_t *entry = *_member;
+
+  if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr,
+             sizeof(struct in6_addr)) < 0)
+    return -1;
+  else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr,
+                  sizeof(struct in6_addr)) > 0)
+    return 1;
+  else
+    return 0;
+}
+
+/** Return 1 if we should collect geoip stats on bridge users, and
+ * include them in our extrainfo descriptor. Else return 0. */
+int
+should_record_bridge_info(const or_options_t *options)
+{
+  return options->BridgeRelay && options->BridgeRecordUsageByCountry;
+}
+
+/** Set up a new list of geoip countries with no countries (yet) set in it,
+ * except for the unknown country.
+ */
+static void
+init_geoip_countries(void)
+{
+  geoip_country_t *geoip_unresolved;
+  geoip_countries = smartlist_new();
+  /* Add a geoip_country_t for requests that could not be resolved to a
+   * country as first element (index 0) to geoip_countries. */
+  geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t));
+  strlcpy(geoip_unresolved->countrycode, "??",
+          sizeof(geoip_unresolved->countrycode));
+  smartlist_add(geoip_countries, geoip_unresolved);
+  country_idxplus1_by_lc_code = strmap_new();
+  strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1));
+}
+
+/** Clear appropriate GeoIP database, based on <b>family</b>, and
+ * reload it from the file <b>filename</b>. Return 0 on success, -1 on
+ * failure.
+ *
+ * Recognized line formats for IPv4 are:
+ *   INTIPLOW,INTIPHIGH,CC
+ * and
+ *   "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME"
+ * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned
+ * integers, and CC is a country code.
+ *
+ * Recognized line format for IPv6 is:
+ *   IPV6LOW,IPV6HIGH,CC
+ * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code.
+ *
+ * It also recognizes, and skips over, blank lines and lines that start
+ * with '#' (comments).
+ */
+int
+geoip_load_file(sa_family_t family, const char *filename)
+{
+  FILE *f;
+  const char *msg = "";
+  const or_options_t *options = get_options();
+  int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO;
+  crypto_digest_t *geoip_digest_env = NULL;
+
+  tor_assert(family == AF_INET || family == AF_INET6);
+
+  if (!(f = tor_fopen_cloexec(filename, "r"))) {
+    log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s.  %s",
+           filename, msg);
+    return -1;
+  }
+  if (!geoip_countries)
+    init_geoip_countries();
+
+  if (family == AF_INET) {
+    if (geoip_ipv4_entries) {
+      SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e,
+                        tor_free(e));
+      smartlist_free(geoip_ipv4_entries);
+    }
+    geoip_ipv4_entries = smartlist_new();
+  } else { /* AF_INET6 */
+    if (geoip_ipv6_entries) {
+      SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e,
+                        tor_free(e));
+      smartlist_free(geoip_ipv6_entries);
+    }
+    geoip_ipv6_entries = smartlist_new();
+  }
+  geoip_digest_env = crypto_digest_new();
+
+  log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.",
+             (family == AF_INET) ? "IPv4" : "IPv6", filename);
+  while (!feof(f)) {
+    char buf[512];
+    if (fgets(buf, (int)sizeof(buf), f) == NULL)
+      break;
+    crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf));
+    /* FFFF track full country name. */
+    geoip_parse_entry(buf, family);
+  }
+  /*XXXX abort and return -1 if no entries/illformed?*/
+  fclose(f);
+
+  /* Sort list and remember file digests so that we can include it in
+   * our extra-info descriptors. */
+  if (family == AF_INET) {
+    smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_);
+    /* Okay, now we need to maybe change our mind about what is in
+     * which country. We do this for IPv4 only since that's what we
+     * store in node->country. */
+    refresh_all_country_info();
+    crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN);
+  } else {
+    /* AF_INET6 */
+    smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_);
+    crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN);
+  }
+  crypto_digest_free(geoip_digest_env);
+
+  return 0;
+}
+
+/** Given an IP address in host order, return a number representing the
+ * country to which that address belongs, -1 for "No geoip information
+ * available", or 0 for the 'unknown country'.  The return value will always
+ * be less than geoip_get_n_countries().  To decode it, call
+ * geoip_get_country_name().
+ */
+STATIC int
+geoip_get_country_by_ipv4(uint32_t ipaddr)
+{
+  geoip_ipv4_entry_t *ent;
+  if (!geoip_ipv4_entries)
+    return -1;
+  ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr,
+                          geoip_ipv4_compare_key_to_entry_);
+  return ent ? (int)ent->country : 0;
+}
+
+/** Given an IPv6 address, return a number representing the country to
+ * which that address belongs, -1 for "No geoip information available", or
+ * 0 for the 'unknown country'.  The return value will always be less than
+ * geoip_get_n_countries().  To decode it, call geoip_get_country_name().
+ */
+STATIC int
+geoip_get_country_by_ipv6(const struct in6_addr *addr)
+{
+  geoip_ipv6_entry_t *ent;
+
+  if (!geoip_ipv6_entries)
+    return -1;
+  ent = smartlist_bsearch(geoip_ipv6_entries, addr,
+                          geoip_ipv6_compare_key_to_entry_);
+  return ent ? (int)ent->country : 0;
+}
+
+/** Given an IP address, return a number representing the country to which
+ * that address belongs, -1 for "No geoip information available", or 0 for
+ * the 'unknown country'.  The return value will always be less than
+ * geoip_get_n_countries().  To decode it, call geoip_get_country_name().
+ */
+MOCK_IMPL(int,
+geoip_get_country_by_addr,(const tor_addr_t *addr))
+{
+  if (tor_addr_family(addr) == AF_INET) {
+    return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr));
+  } else if (tor_addr_family(addr) == AF_INET6) {
+    return geoip_get_country_by_ipv6(tor_addr_to_in6(addr));
+  } else {
+    return -1;
+  }
+}
+
+/** Return the number of countries recognized by the GeoIP country list. */
+MOCK_IMPL(int,
+geoip_get_n_countries,(void))
+{
+  if (!geoip_countries)
+    init_geoip_countries();
+  return (int) smartlist_len(geoip_countries);
+}
+
+/** Return the two-letter country code associated with the number <b>num</b>,
+ * or "??" for an unknown value. */
+const char *
+geoip_get_country_name(country_t num)
+{
+  if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) {
+    geoip_country_t *c = smartlist_get(geoip_countries, num);
+    return c->countrycode;
+  } else
+    return "??";
+}
+
+/** Return true iff we have loaded a GeoIP database.*/
+MOCK_IMPL(int,
+geoip_is_loaded,(sa_family_t family))
+{
+  tor_assert(family == AF_INET || family == AF_INET6);
+  if (geoip_countries == NULL)
+    return 0;
+  if (family == AF_INET)
+    return geoip_ipv4_entries != NULL;
+  else                          /* AF_INET6 */
+    return geoip_ipv6_entries != NULL;
+}
+
+/** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The
+ * result does not need to be deallocated, but will be overwritten by the
+ * next call of hex_str(). */
+const char *
+geoip_db_digest(sa_family_t family)
+{
+  tor_assert(family == AF_INET || family == AF_INET6);
+  if (family == AF_INET)
+    return hex_str(geoip_digest, DIGEST_LEN);
+  else                          /* AF_INET6 */
+    return hex_str(geoip6_digest, DIGEST_LEN);
+}
+
+/** Largest allowable value for last_seen_in_minutes.  (It's a 30-bit field,
+ * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
+ */
+#define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
+
+/** Map from client IP address to last time seen. */
+static HT_HEAD(clientmap, clientmap_entry_t) client_history =
+     HT_INITIALIZER();
+
+/** Hashtable helper: compute a hash of a clientmap_entry_t. */
+static inline unsigned
+clientmap_entry_hash(const clientmap_entry_t *a)
+{
+  unsigned h = (unsigned) tor_addr_hash(&a->addr);
+
+  if (a->transport_name)
+    h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name));
+
+  return h;
+}
+/** Hashtable helper: compare two clientmap_entry_t values for equality. */
+static inline int
+clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
+{
+  if (strcmp_opt(a->transport_name, b->transport_name))
+    return 0;
+
+  return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) &&
+         a->action == b->action;
+}
+
+HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
+             clientmap_entries_eq)
+HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
+             clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+
+#define clientmap_entry_free(ent) \
+  FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
+
+/** Return the size of a client map entry. */
+static inline size_t
+clientmap_entry_size(const clientmap_entry_t *ent)
+{
+  tor_assert(ent);
+  return (sizeof(clientmap_entry_t) +
+          (ent->transport_name ? strlen(ent->transport_name) : 0));
+}
+
+/** Free all storage held by <b>ent</b>. */
+static void
+clientmap_entry_free_(clientmap_entry_t *ent)
+{
+  if (!ent)
+    return;
+
+  /* This entry is about to be freed so pass it to the DoS subsystem to see if
+   * any actions can be taken about it. */
+  dos_geoip_entry_about_to_free(ent);
+  geoip_decrement_client_history_cache_size(clientmap_entry_size(ent));
+
+  tor_free(ent->transport_name);
+  tor_free(ent);
+}
+
+/* Return a newly allocated clientmap entry with the given action and address
+ * that are mandatory. The transport_name can be optional. This can't fail. */
+static clientmap_entry_t *
+clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr,
+                    const char *transport_name)
+{
+  clientmap_entry_t *entry;
+
+  tor_assert(action == GEOIP_CLIENT_CONNECT ||
+             action == GEOIP_CLIENT_NETWORKSTATUS);
+  tor_assert(addr);
+
+  entry = tor_malloc_zero(sizeof(clientmap_entry_t));
+  entry->action = action;
+  tor_addr_copy(&entry->addr, addr);
+  if (transport_name) {
+    entry->transport_name = tor_strdup(transport_name);
+  }
+
+  /* Allocated and initialized, note down its size for the OOM handler. */
+  geoip_increment_client_history_cache_size(clientmap_entry_size(entry));
+
+  return entry;
+}
+
+/** Clear history of connecting clients used by entry and bridge stats. */
+static void
+client_history_clear(void)
+{
+  clientmap_entry_t **ent, **next, *this;
+  for (ent = HT_START(clientmap, &client_history); ent != NULL;
+       ent = next) {
+    if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
+      this = *ent;
+      next = HT_NEXT_RMV(clientmap, &client_history, ent);
+      clientmap_entry_free(this);
+    } else {
+      next = HT_NEXT(clientmap, &client_history, ent);
+    }
+  }
+}
+
+/** Note that we've seen a client connect from the IP <b>addr</b>
+ * at time <b>now</b>. Ignored by all but bridges and directories if
+ * configured accordingly. */
+void
+geoip_note_client_seen(geoip_client_action_t action,
+                       const tor_addr_t *addr,
+                       const char *transport_name,
+                       time_t now)
+{
+  const or_options_t *options = get_options();
+  clientmap_entry_t *ent;
+
+  if (action == GEOIP_CLIENT_CONNECT) {
+    /* Only remember statistics if the DoS mitigation subsystem is enabled. If
+     * not, only if as entry guard or as bridge. */
+    if (!dos_enabled()) {
+      if (!options->EntryStatistics && !should_record_bridge_info(options)) {
+        return;
+      }
+    }
+  } else {
+    /* Only gather directory-request statistics if configured, and
+     * forcibly disable them on bridge authorities. */
+    if (!options->DirReqStatistics || options->BridgeAuthoritativeDir)
+      return;
+  }
+
+  log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.",
+            safe_str_client(fmt_addr((addr))),
+            transport_name ? transport_name : "<no transport>");
+
+  ent = geoip_lookup_client(addr, transport_name, action);
+  if (! ent) {
+    ent = clientmap_entry_new(action, addr, transport_name);
+    HT_INSERT(clientmap, &client_history, ent);
+  }
+  if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
+    ent->last_seen_in_minutes = (unsigned)(now/60);
+  else
+    ent->last_seen_in_minutes = 0;
+
+  if (action == GEOIP_CLIENT_NETWORKSTATUS) {
+    int country_idx = geoip_get_country_by_addr(addr);
+    if (country_idx < 0)
+      country_idx = 0; /** unresolved requests are stored at index 0. */
+    if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) {
+      geoip_country_t *country = smartlist_get(geoip_countries, country_idx);
+      ++country->n_v3_ns_requests;
+    }
+  }
+}
+
+/** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
+ * older than a certain time. */
+static int
+remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff)
+{
+  time_t cutoff = *(time_t*)_cutoff / 60;
+  if (ent->last_seen_in_minutes < cutoff) {
+    clientmap_entry_free(ent);
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+/** Forget about all clients that haven't connected since <b>cutoff</b>. */
+void
+geoip_remove_old_clients(time_t cutoff)
+{
+  clientmap_HT_FOREACH_FN(&client_history,
+                          remove_old_client_helper_,
+                          &cutoff);
+}
+
+/* Return a client entry object matching the given address, transport name and
+ * geoip action from the clientmap. NULL if not found. The transport_name can
+ * be NULL. */
+clientmap_entry_t *
+geoip_lookup_client(const tor_addr_t *addr, const char *transport_name,
+                    geoip_client_action_t action)
+{
+  clientmap_entry_t lookup;
+
+  tor_assert(addr);
+
+  /* We always look for a client connection with no transport. */
+  tor_addr_copy(&lookup.addr, addr);
+  lookup.action = action;
+  lookup.transport_name = (char *) transport_name;
+
+  return HT_FIND(clientmap, &client_history, &lookup);
+}
+
+/* Cleanup client entries older than the cutoff. Used for the OOM. Return the
+ * number of bytes freed. If 0 is returned, nothing was freed. */
+static size_t
+oom_clean_client_entries(time_t cutoff)
+{
+  size_t bytes = 0;
+  clientmap_entry_t **ent, **ent_next;
+
+  for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
+    clientmap_entry_t *entry = *ent;
+    if (entry->last_seen_in_minutes < (cutoff / 60)) {
+      ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
+      bytes += clientmap_entry_size(entry);
+      clientmap_entry_free(entry);
+    } else {
+      ent_next = HT_NEXT(clientmap, &client_history, ent);
+    }
+  }
+  return bytes;
+}
+
+/* Below this minimum lifetime, the OOM won't cleanup any entries. */
+#define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
+/* The OOM moves the cutoff by that much every run. */
+#define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
+
+/* Cleanup the geoip client history cache called from the OOM handler. Return
+ * the amount of bytes removed. This can return a value below or above
+ * min_remove_bytes but will stop as oon as the min_remove_bytes has been
+ * reached. */
+size_t
+geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes)
+{
+  time_t k;
+  size_t bytes_removed = 0;
+
+  /* Our OOM handler called with 0 bytes to remove is a code flow error. */
+  tor_assert(min_remove_bytes != 0);
+
+  /* Set k to the initial cutoff of an entry. We then going to move it by step
+   * to try to remove as much as we can. */
+  k = WRITE_STATS_INTERVAL;
+
+  do {
+    time_t cutoff;
+
+    /* If k has reached the minimum lifetime, we have to stop else we might
+     * remove every single entries which would be pretty bad for the DoS
+     * mitigation subsystem if by just filling the geoip cache, it was enough
+     * to trigger the OOM and clean every single entries. */
+    if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
+      break;
+    }
+
+    cutoff = now - k;
+    bytes_removed += oom_clean_client_entries(cutoff);
+    k -= GEOIP_CLIENT_CACHE_OOM_STEP;
+  } while (bytes_removed < min_remove_bytes);
+
+  return bytes_removed;
+}
+
+/* Return the total size in bytes of the client history cache. */
+size_t
+geoip_client_cache_total_allocation(void)
+{
+  return geoip_client_history_cache_size;
+}
+
+/** How many responses are we giving to clients requesting v3 network
+ * statuses? */
+static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
+
+/** Note that we've rejected a client's request for a v3 network status
+ * for reason <b>reason</b> at time <b>now</b>. */
+void
+geoip_note_ns_response(geoip_ns_response_t response)
+{
+  static int arrays_initialized = 0;
+  if (!get_options()->DirReqStatistics)
+    return;
+  if (!arrays_initialized) {
+    memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
+    arrays_initialized = 1;
+  }
+  tor_assert(response < GEOIP_NS_RESPONSE_NUM);
+  ns_v3_responses[response]++;
+}
+
+/** Do not mention any country from which fewer than this number of IPs have
+ * connected.  This conceivably avoids reporting information that could
+ * deanonymize users, though analysis is lacking. */
+#define MIN_IPS_TO_NOTE_COUNTRY 1
+/** Do not report any geoip data at all if we have fewer than this number of
+ * IPs to report about. */
+#define MIN_IPS_TO_NOTE_ANYTHING 1
+/** When reporting geoip data about countries, round up to the nearest
+ * multiple of this value. */
+#define IP_GRANULARITY 8
+
+/** Helper type: used to sort per-country totals by value. */
+typedef struct c_hist_t {
+  char country[3]; /**< Two-letter country code. */
+  unsigned total; /**< Total IP addresses seen in this country. */
+} c_hist_t;
+
+/** Sorting helper: return -1, 1, or 0 based on comparison of two
+ * geoip_ipv4_entry_t.  Sort in descending order of total, and then by country
+ * code. */
+static int
+c_hist_compare_(const void **_a, const void **_b)
+{
+  const c_hist_t *a = *_a, *b = *_b;
+  if (a->total > b->total)
+    return -1;
+  else if (a->total < b->total)
+    return 1;
+  else
+    return strcmp(a->country, b->country);
+}
+
+/** When there are incomplete directory requests at the end of a 24-hour
+ * period, consider those requests running for longer than this timeout as
+ * failed, the others as still running. */
+#define DIRREQ_TIMEOUT (10*60)
+
+/** Entry in a map from either chan->global_identifier for direct requests
+ * or a unique circuit identifier for tunneled requests to request time,
+ * response size, and completion time of a network status request. Used to
+ * measure download times of requests to derive average client
+ * bandwidths. */
+typedef struct dirreq_map_entry_t {
+  HT_ENTRY(dirreq_map_entry_t) node;
+  /** Unique identifier for this network status request; this is either the
+   * chan->global_identifier of the dir channel (direct request) or a new
+   * locally unique identifier of a circuit (tunneled request). This ID is
+   * only unique among other direct or tunneled requests, respectively. */
+  uint64_t dirreq_id;
+  unsigned int state:3; /**< State of this directory request. */
+  unsigned int type:1; /**< Is this a direct or a tunneled request? */
+  unsigned int completed:1; /**< Is this request complete? */
+  /** When did we receive the request and started sending the response? */
+  struct timeval request_time;
+  size_t response_size; /**< What is the size of the response in bytes? */
+  struct timeval completion_time; /**< When did the request succeed? */
+} dirreq_map_entry_t;
+
+/** Map of all directory requests asking for v2 or v3 network statuses in
+ * the current geoip-stats interval. Values are
+ * of type *<b>dirreq_map_entry_t</b>. */
+static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
+     HT_INITIALIZER();
+
+static int
+dirreq_map_ent_eq(const dirreq_map_entry_t *a,
+                  const dirreq_map_entry_t *b)
+{
+  return a->dirreq_id == b->dirreq_id && a->type == b->type;
+}
+
+/* DOCDOC dirreq_map_ent_hash */
+static unsigned
+dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
+{
+  unsigned u = (unsigned) entry->dirreq_id;
+  u += entry->type << 20;
+  return u;
+}
+
+HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
+             dirreq_map_ent_eq)
+HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
+             dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/** Helper: Put <b>entry</b> into map of directory requests using
+ * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
+ * already an entry for that key, print out a BUG warning and return. */
+static void
+dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type,
+               uint64_t dirreq_id)
+{
+  dirreq_map_entry_t *old_ent;
+  tor_assert(entry->type == type);
+  tor_assert(entry->dirreq_id == dirreq_id);
+
+  /* XXXX we could switch this to HT_INSERT some time, since it seems that
+   * this bug doesn't happen. But since this function doesn't seem to be
+   * critical-path, it's sane to leave it alone. */
+  old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
+  if (old_ent && old_ent != entry) {
+    log_warn(LD_BUG, "Error when putting directory request into local "
+             "map. There was already an entry for the same identifier.");
+    return;
+  }
+}
+
+/** Helper: Look up and return an entry in the map of directory requests
+ * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
+ * is no such entry, return NULL. */
+static dirreq_map_entry_t *
+dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id)
+{
+  dirreq_map_entry_t lookup;
+  lookup.type = type;
+  lookup.dirreq_id = dirreq_id;
+  return HT_FIND(dirreqmap, &dirreq_map, &lookup);
+}
+
+/** Note that an either direct or tunneled (see <b>type</b>) directory
+ * request for a v3 network status with unique ID <b>dirreq_id</b> of size
+ * <b>response_size</b> has started. */
+void
+geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
+                   dirreq_type_t type)
+{
+  dirreq_map_entry_t *ent;
+  if (!get_options()->DirReqStatistics)
+    return;
+  ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
+  ent->dirreq_id = dirreq_id;
+  tor_gettimeofday(&ent->request_time);
+  ent->response_size = response_size;
+  ent->type = type;
+  dirreq_map_put_(ent, type, dirreq_id);
+}
+
+/** Change the state of the either direct or tunneled (see <b>type</b>)
+ * directory request with <b>dirreq_id</b> to <b>new_state</b> and
+ * possibly mark it as completed. If no entry can be found for the given
+ * key parts (e.g., if this is a directory request that we are not
+ * measuring, or one that was started in the previous measurement period),
+ * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
+void
+geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
+                          dirreq_state_t new_state)
+{
+  dirreq_map_entry_t *ent;
+  if (!get_options()->DirReqStatistics)
+    return;
+  ent = dirreq_map_get_(type, dirreq_id);
+  if (!ent)
+    return;
+  if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
+    return;
+  if (new_state - 1 != ent->state)
+    return;
+  ent->state = new_state;
+  if ((type == DIRREQ_DIRECT &&
+         new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
+      (type == DIRREQ_TUNNELED &&
+         new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
+    tor_gettimeofday(&ent->completion_time);
+    ent->completed = 1;
+  }
+}
+
+/** Return the bridge-ip-transports string that should be inserted in
+ *  our extra-info descriptor. Return NULL if the bridge-ip-transports
+ *  line should be empty.  */
+char *
+geoip_get_transport_history(void)
+{
+  unsigned granularity = IP_GRANULARITY;
+  /** String hash table (name of transport) -> (number of users). */
+  strmap_t *transport_counts = strmap_new();
+
+  /** Smartlist that contains copies of the names of the transports
+      that have been used. */
+  smartlist_t *transports_used = smartlist_new();
+
+  /* Special string to signify that no transport was used for this
+     connection. Pluggable transport names can't have symbols in their
+     names, so this string will never collide with a real transport. */
+  static const char* no_transport_str = "<OR>";
+
+  clientmap_entry_t **ent;
+  smartlist_t *string_chunks = smartlist_new();
+  char *the_string = NULL;
+
+  /* If we haven't seen any clients yet, return NULL. */
+  if (HT_EMPTY(&client_history))
+    goto done;
+
+  /** We do the following steps to form the transport history string:
+   *  a) Foreach client that uses a pluggable transport, we increase the
+   *  times that transport was used by one. If the client did not use
+   *  a transport, we increase the number of times someone connected
+   *  without obfuscation.
+   *  b) Foreach transport we observed, we write its transport history
+   *  string and push it to string_chunks. So, for example, if we've
+   *  seen 665 obfs2 clients, we write "obfs2=665".
+   *  c) We concatenate string_chunks to form the final string.
+   */
+
+  log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.",
+            HT_SIZE(&client_history));
+
+  /* Loop through all clients. */
+  HT_FOREACH(ent, clientmap, &client_history) {
+    uintptr_t val;
+    void *ptr;
+    const char *transport_name = (*ent)->transport_name;
+    if (!transport_name)
+      transport_name = no_transport_str;
+
+    /* Increase the count for this transport name. */
+    ptr = strmap_get(transport_counts, transport_name);
+    val = (uintptr_t)ptr;
+    val++;
+    ptr = (void*)val;
+    strmap_set(transport_counts, transport_name, ptr);
+
+    /* If it's the first time we see this transport, note it. */
+    if (val == 1)
+      smartlist_add_strdup(transports_used, transport_name);
+
+    log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
+              "I've now seen %d clients.",
+              safe_str_client(fmt_addr(&(*ent)->addr)),
+              transport_name ? transport_name : "<no transport>",
+              (int)val);
+  }
+
+  /* Sort the transport names (helps with unit testing). */
+  smartlist_sort_strings(transports_used);
+
+  /* Loop through all seen transports. */
+  SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) {
+    void *transport_count_ptr = strmap_get(transport_counts, transport_name);
+    uintptr_t transport_count = (uintptr_t) transport_count_ptr;
+
+    log_debug(LD_GENERAL, "We got %"PRIu64" clients with transport '%s'.",
+              ((uint64_t)transport_count), transport_name);
+
+    smartlist_add_asprintf(string_chunks, "%s=%"PRIu64,
+                           transport_name,
+                           (round_uint64_to_next_multiple_of(
+                                               (uint64_t)transport_count,
+                                               granularity)));
+  } SMARTLIST_FOREACH_END(transport_name);
+
+  the_string = smartlist_join_strings(string_chunks, ",", 0, NULL);
+
+  log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string);
+
+ done:
+  strmap_free(transport_counts, NULL);
+  SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s));
+  smartlist_free(transports_used);
+  SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s));
+  smartlist_free(string_chunks);
+
+  return the_string;
+}
+
+/** Return a newly allocated comma-separated string containing statistics
+ * on network status downloads. The string contains the number of completed
+ * requests, timeouts, and still running requests as well as the download
+ * times by deciles and quartiles. Return NULL if we have not observed
+ * requests for long enough. */
+static char *
+geoip_get_dirreq_history(dirreq_type_t type)
+{
+  char *result = NULL;
+  buf_t *buf = NULL;
+  smartlist_t *dirreq_completed = NULL;
+  uint32_t complete = 0, timeouts = 0, running = 0;
+  dirreq_map_entry_t **ptr, **next;
+  struct timeval now;
+
+  tor_gettimeofday(&now);
+  dirreq_completed = smartlist_new();
+  for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
+    dirreq_map_entry_t *ent = *ptr;
+    if (ent->type != type) {
+      next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
+      continue;
+    } else {
+      if (ent->completed) {
+        smartlist_add(dirreq_completed, ent);
+        complete++;
+        next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
+      } else {
+        if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
+          timeouts++;
+        else
+          running++;
+        next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
+        tor_free(ent);
+      }
+    }
+  }
+#define DIR_REQ_GRANULARITY 4
+  complete = round_uint32_to_next_multiple_of(complete,
+                                              DIR_REQ_GRANULARITY);
+  timeouts = round_uint32_to_next_multiple_of(timeouts,
+                                              DIR_REQ_GRANULARITY);
+  running = round_uint32_to_next_multiple_of(running,
+                                             DIR_REQ_GRANULARITY);
+  buf = buf_new_with_capacity(1024);
+  buf_add_printf(buf, "complete=%u,timeout=%u,"
+                 "running=%u", complete, timeouts, running);
+
+#define MIN_DIR_REQ_RESPONSES 16
+  if (complete >= MIN_DIR_REQ_RESPONSES) {
+    uint32_t *dltimes;
+    /* We may have rounded 'completed' up.  Here we want to use the
+     * real value. */
+    complete = smartlist_len(dirreq_completed);
+    dltimes = tor_calloc(complete, sizeof(uint32_t));
+    SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
+      uint32_t bytes_per_second;
+      uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time,
+                                               &ent->completion_time);
+      if (time_diff == 0)
+        time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible
+                        * by law of nature or something, but a millisecond
+                        * is a bit greater than "instantly" */
+      bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff);
+      dltimes[ent_sl_idx] = bytes_per_second;
+    } SMARTLIST_FOREACH_END(ent);
+    median_uint32(dltimes, complete); /* sorts as a side effect. */
+    buf_add_printf(buf,
+                           ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
+                           "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
+                           dltimes[0],
+                           dltimes[1*complete/10-1],
+                           dltimes[2*complete/10-1],
+                           dltimes[1*complete/4-1],
+                           dltimes[3*complete/10-1],
+                           dltimes[4*complete/10-1],
+                           dltimes[5*complete/10-1],
+                           dltimes[6*complete/10-1],
+                           dltimes[7*complete/10-1],
+                           dltimes[3*complete/4-1],
+                           dltimes[8*complete/10-1],
+                           dltimes[9*complete/10-1],
+                           dltimes[complete-1]);
+    tor_free(dltimes);
+  }
+
+  result = buf_extract(buf, NULL);
+
+  SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
+                    tor_free(ent));
+  smartlist_free(dirreq_completed);
+  buf_free(buf);
+  return result;
+}
+
+/** Store a newly allocated comma-separated string in
+ * *<a>country_str</a> containing entries for all the countries from
+ * which we've seen enough clients connect as a bridge, directory
+ * server, or entry guard. The entry format is cc=num where num is the
+ * number of IPs we've seen connecting from that country, and cc is a
+ * lowercased country code. *<a>country_str</a> is set to NULL if
+ * we're not ready to export per country data yet.
+ *
+ * Store a newly allocated comma-separated string in <a>ipver_str</a>
+ * containing entries for clients connecting over IPv4 and IPv6. The
+ * format is family=num where num is the nubmer of IPs we've seen
+ * connecting over that protocol family, and family is 'v4' or 'v6'.
+ *
+ * Return 0 on success and -1 if we're missing geoip data. */
+int
+geoip_get_client_history(geoip_client_action_t action,
+                         char **country_str, char **ipver_str)
+{
+  unsigned granularity = IP_GRANULARITY;
+  smartlist_t *entries = NULL;
+  int n_countries = geoip_get_n_countries();
+  int i;
+  clientmap_entry_t **cm_ent;
+  unsigned *counts = NULL;
+  unsigned total = 0;
+  unsigned ipv4_count = 0, ipv6_count = 0;
+
+  if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6))
+    return -1;
+
+  counts = tor_calloc(n_countries, sizeof(unsigned));
+  HT_FOREACH(cm_ent, clientmap, &client_history) {
+    int country;
+    if ((*cm_ent)->action != (int)action)
+      continue;
+    country = geoip_get_country_by_addr(&(*cm_ent)->addr);
+    if (country < 0)
+      country = 0; /** unresolved requests are stored at index 0. */
+    tor_assert(0 <= country && country < n_countries);
+    ++counts[country];
+    ++total;
+    switch (tor_addr_family(&(*cm_ent)->addr)) {
+    case AF_INET:
+      ipv4_count++;
+      break;
+    case AF_INET6:
+      ipv6_count++;
+      break;
+    }
+  }
+  if (ipver_str) {
+    smartlist_t *chunks = smartlist_new();
+    smartlist_add_asprintf(chunks, "v4=%u",
+                           round_to_next_multiple_of(ipv4_count, granularity));
+    smartlist_add_asprintf(chunks, "v6=%u",
+                           round_to_next_multiple_of(ipv6_count, granularity));
+    *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL);
+    SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
+    smartlist_free(chunks);
+  }
+
+  /* Don't record per country data if we haven't seen enough IPs. */
+  if (total < MIN_IPS_TO_NOTE_ANYTHING) {
+    tor_free(counts);
+    if (country_str)
+      *country_str = NULL;
+    return 0;
+  }
+
+  /* Make a list of c_hist_t */
+  entries = smartlist_new();
+  for (i = 0; i < n_countries; ++i) {
+    unsigned c = counts[i];
+    const char *countrycode;
+    c_hist_t *ent;
+    /* Only report a country if it has a minimum number of IPs. */
+    if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
+      c = round_to_next_multiple_of(c, granularity);
+      countrycode = geoip_get_country_name(i);
+      ent = tor_malloc(sizeof(c_hist_t));
+      strlcpy(ent->country, countrycode, sizeof(ent->country));
+      ent->total = c;
+      smartlist_add(entries, ent);
+    }
+  }
+  /* Sort entries. Note that we must do this _AFTER_ rounding, or else
+   * the sort order could leak info. */
+  smartlist_sort(entries, c_hist_compare_);
+
+  if (country_str) {
+    smartlist_t *chunks = smartlist_new();
+    SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
+        smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total);
+      });
+    *country_str = smartlist_join_strings(chunks, ",", 0, NULL);
+    SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
+    smartlist_free(chunks);
+  }
+
+  SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
+  smartlist_free(entries);
+  tor_free(counts);
+
+  return 0;
+}
+
+/** Return a newly allocated string holding the per-country request history
+ * for v3 network statuses in a format suitable for an extra-info document,
+ * or NULL on failure. */
+char *
+geoip_get_request_history(void)
+{
+  smartlist_t *entries, *strings;
+  char *result;
+  unsigned granularity = IP_GRANULARITY;
+
+  if (!geoip_countries)
+    return NULL;
+
+  entries = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) {
+      uint32_t tot = 0;
+      c_hist_t *ent;
+      tot = c->n_v3_ns_requests;
+      if (!tot)
+        continue;
+      ent = tor_malloc_zero(sizeof(c_hist_t));
+      strlcpy(ent->country, c->countrycode, sizeof(ent->country));
+      ent->total = round_to_next_multiple_of(tot, granularity);
+      smartlist_add(entries, ent);
+  } SMARTLIST_FOREACH_END(c);
+  smartlist_sort(entries, c_hist_compare_);
+
+  strings = smartlist_new();
+  SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
+      smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total);
+  });
+  result = smartlist_join_strings(strings, ",", 0, NULL);
+  SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
+  SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
+  smartlist_free(strings);
+  smartlist_free(entries);
+  return result;
+}
+
+/** Start time of directory request stats or 0 if we're not collecting
+ * directory request statistics. */
+static time_t start_of_dirreq_stats_interval;
+
+/** Initialize directory request stats. */
+void
+geoip_dirreq_stats_init(time_t now)
+{
+  start_of_dirreq_stats_interval = now;
+}
+
+/** Reset counters for dirreq stats. */
+void
+geoip_reset_dirreq_stats(time_t now)
+{
+  SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
+      c->n_v3_ns_requests = 0;
+  });
+  {
+    clientmap_entry_t **ent, **next, *this;
+    for (ent = HT_START(clientmap, &client_history); ent != NULL;
+         ent = next) {
+      if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) {
+        this = *ent;
+        next = HT_NEXT_RMV(clientmap, &client_history, ent);
+        clientmap_entry_free(this);
+      } else {
+        next = HT_NEXT(clientmap, &client_history, ent);
+      }
+    }
+  }
+  memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
+  {
+    dirreq_map_entry_t **ent, **next, *this;
+    for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
+      this = *ent;
+      next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
+      tor_free(this);
+    }
+  }
+  start_of_dirreq_stats_interval = now;
+}
+
+/** Stop collecting directory request stats in a way that we can re-start
+ * doing so in geoip_dirreq_stats_init(). */
+void
+geoip_dirreq_stats_term(void)
+{
+  geoip_reset_dirreq_stats(0);
+}
+
+/** Return a newly allocated string containing the dirreq statistics
+ * until <b>now</b>, or NULL if we're not collecting dirreq stats. Caller
+ * must ensure start_of_dirreq_stats_interval is in the past. */
+char *
+geoip_format_dirreq_stats(time_t now)
+{
+  char t[ISO_TIME_LEN+1];
+  int i;
+  char *v3_ips_string = NULL, *v3_reqs_string = NULL,
+       *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL;
+  char *result = NULL;
+
+  if (!start_of_dirreq_stats_interval)
+    return NULL; /* Not initialized. */
+
+  tor_assert(now >= start_of_dirreq_stats_interval);
+
+  format_iso_time(t, now);
+  geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL);
+  v3_reqs_string = geoip_get_request_history();
+
+#define RESPONSE_GRANULARITY 8
+  for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
+    ns_v3_responses[i] = round_uint32_to_next_multiple_of(
+                               ns_v3_responses[i], RESPONSE_GRANULARITY);
+  }
+#undef RESPONSE_GRANULARITY
+
+  v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
+  v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
+
+  /* Put everything together into a single string. */
+  tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n"
+              "dirreq-v3-ips %s\n"
+              "dirreq-v3-reqs %s\n"
+              "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
+                   "not-found=%u,not-modified=%u,busy=%u\n"
+              "dirreq-v3-direct-dl %s\n"
+              "dirreq-v3-tunneled-dl %s\n",
+              t,
+              (unsigned) (now - start_of_dirreq_stats_interval),
+              v3_ips_string ? v3_ips_string : "",
+              v3_reqs_string ? v3_reqs_string : "",
+              ns_v3_responses[GEOIP_SUCCESS],
+              ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
+              ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
+              ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
+              ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
+              ns_v3_responses[GEOIP_REJECT_BUSY],
+              v3_direct_dl_string ? v3_direct_dl_string : "",
+              v3_tunneled_dl_string ? v3_tunneled_dl_string : "");
+
+  /* Free partial strings. */
+  tor_free(v3_ips_string);
+  tor_free(v3_reqs_string);
+  tor_free(v3_direct_dl_string);
+  tor_free(v3_tunneled_dl_string);
+
+  return result;
+}
+
+/** If 24 hours have passed since the beginning of the current dirreq
+ * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats
+ * (possibly overwriting an existing file) and reset counters.  Return
+ * when we would next want to write dirreq stats or 0 if we never want to
+ * write. */
+time_t
+geoip_dirreq_stats_write(time_t now)
+{
+  char *str = NULL;
+
+  if (!start_of_dirreq_stats_interval)
+    return 0; /* Not initialized. */
+  if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
+    goto done; /* Not ready to write. */
+
+  /* Discard all items in the client history that are too old. */
+  geoip_remove_old_clients(start_of_dirreq_stats_interval);
+
+  /* Generate history string .*/
+  str = geoip_format_dirreq_stats(now);
+  if (! str)
+    goto done;
+
+  /* Write dirreq-stats string to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics");
+    /* Reset measurement interval start. */
+    geoip_reset_dirreq_stats(now);
+  }
+
+ done:
+  tor_free(str);
+  return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Start time of bridge stats or 0 if we're not collecting bridge
+ * statistics. */
+static time_t start_of_bridge_stats_interval;
+
+/** Initialize bridge stats. */
+void
+geoip_bridge_stats_init(time_t now)
+{
+  start_of_bridge_stats_interval = now;
+}
+
+/** Stop collecting bridge stats in a way that we can re-start doing so in
+ * geoip_bridge_stats_init(). */
+void
+geoip_bridge_stats_term(void)
+{
+  client_history_clear();
+  start_of_bridge_stats_interval = 0;
+}
+
+/** Validate a bridge statistics string as it would be written to a
+ * current extra-info descriptor. Return 1 if the string is valid and
+ * recent enough, or 0 otherwise. */
+static int
+validate_bridge_stats(const char *stats_str, time_t now)
+{
+  char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
+       *eos;
+
+  const char *BRIDGE_STATS_END = "bridge-stats-end ";
+  const char *BRIDGE_IPS = "bridge-ips ";
+  const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
+  const char *BRIDGE_TRANSPORTS = "bridge-ip-transports ";
+  const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n";
+  const char *tmp;
+  time_t stats_end_time;
+  int seconds;
+  tor_assert(stats_str);
+
+  /* Parse timestamp and number of seconds from
+    "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
+  tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
+  if (!tmp)
+    return 0;
+  tmp += strlen(BRIDGE_STATS_END);
+
+  if (strlen(tmp) < ISO_TIME_LEN + 6)
+    return 0;
+  strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
+  if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
+    return 0;
+  if (stats_end_time < now - (25*60*60) ||
+      stats_end_time > now + (1*60*60))
+    return 0;
+  seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
+  if (!eos || seconds < 23*60*60)
+    return 0;
+  format_iso_time(stats_start_str, stats_end_time - seconds);
+
+  /* Parse: "bridge-ips CC=N,CC=N,..." */
+  tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
+  if (!tmp) {
+    /* Look if there is an empty "bridge-ips" line */
+    tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
+    if (!tmp)
+      return 0;
+  }
+
+  /* Parse: "bridge-ip-transports PT=N,PT=N,..." */
+  tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS);
+  if (!tmp) {
+    /* Look if there is an empty "bridge-ip-transports" line */
+    tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE);
+    if (!tmp)
+      return 0;
+  }
+
+  return 1;
+}
+
+/** Most recent bridge statistics formatted to be written to extra-info
+ * descriptors. */
+static char *bridge_stats_extrainfo = NULL;
+
+/** Return a newly allocated string holding our bridge usage stats by country
+ * in a format suitable for inclusion in an extrainfo document. Return NULL on
+ * failure.  */
+char *
+geoip_format_bridge_stats(time_t now)
+{
+  char *out = NULL;
+  char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL;
+  long duration = now - start_of_bridge_stats_interval;
+  char written[ISO_TIME_LEN+1];
+
+  if (duration < 0)
+    return NULL;
+  if (!start_of_bridge_stats_interval)
+    return NULL; /* Not initialized. */
+
+  format_iso_time(written, now);
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
+  transport_data = geoip_get_transport_history();
+
+  tor_asprintf(&out,
+               "bridge-stats-end %s (%ld s)\n"
+               "bridge-ips %s\n"
+               "bridge-ip-versions %s\n"
+               "bridge-ip-transports %s\n",
+               written, duration,
+               country_data ? country_data : "",
+               ipver_data ? ipver_data : "",
+               transport_data ? transport_data : "");
+  tor_free(country_data);
+  tor_free(ipver_data);
+  tor_free(transport_data);
+
+  return out;
+}
+
+/** Return a newly allocated string holding our bridge usage stats by country
+ * in a format suitable for the answer to a controller request. Return NULL on
+ * failure.  */
+static char *
+format_bridge_stats_controller(time_t now)
+{
+  char *out = NULL, *country_data = NULL, *ipver_data = NULL;
+  char started[ISO_TIME_LEN+1];
+  (void) now;
+
+  format_iso_time(started, start_of_bridge_stats_interval);
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
+
+  tor_asprintf(&out,
+               "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
+               started,
+               country_data ? country_data : "",
+               ipver_data ? ipver_data : "");
+  tor_free(country_data);
+  tor_free(ipver_data);
+  return out;
+}
+
+/** Return a newly allocated string holding our bridge usage stats by
+ * country in a format suitable for inclusion in our heartbeat
+ * message. Return NULL on failure.  */
+char *
+format_client_stats_heartbeat(time_t now)
+{
+  const int n_hours = 6;
+  char *out = NULL;
+  int n_clients = 0;
+  clientmap_entry_t **ent;
+  unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 );
+
+  if (!start_of_bridge_stats_interval)
+    return NULL; /* Not initialized. */
+
+  /* count unique IPs */
+  HT_FOREACH(ent, clientmap, &client_history) {
+    /* only count directly connecting clients */
+    if ((*ent)->action != GEOIP_CLIENT_CONNECT)
+      continue;
+    if ((*ent)->last_seen_in_minutes < cutoff)
+      continue;
+    n_clients++;
+  }
+
+  tor_asprintf(&out, "Heartbeat: "
+               "In the last %d hours, I have seen %d unique clients.",
+               n_hours,
+               n_clients);
+
+  return out;
+}
+
+/** Write bridge statistics to $DATADIR/stats/bridge-stats and return
+ * when we should next try to write statistics. */
+time_t
+geoip_bridge_stats_write(time_t now)
+{
+  char *val = NULL;
+
+  /* Check if 24 hours have passed since starting measurements. */
+  if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
+    return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
+
+  /* Discard all items in the client history that are too old. */
+  geoip_remove_old_clients(start_of_bridge_stats_interval);
+
+  /* Generate formatted string */
+  val = geoip_format_bridge_stats(now);
+  if (val == NULL)
+    goto done;
+
+  /* Update the stored value. */
+  tor_free(bridge_stats_extrainfo);
+  bridge_stats_extrainfo = val;
+  start_of_bridge_stats_interval = now;
+
+  /* Write it to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "bridge-stats",
+                         bridge_stats_extrainfo, "bridge statistics");
+
+    /* Tell the controller, "hey, there are clients!" */
+    {
+      char *controller_str = format_bridge_stats_controller(now);
+      if (controller_str)
+        control_event_clients_seen(controller_str);
+      tor_free(controller_str);
+    }
+  }
+
+ done:
+  return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Try to load the most recent bridge statistics from disk, unless we
+ * have finished a measurement interval lately, and check whether they
+ * are still recent enough. */
+static void
+load_bridge_stats(time_t now)
+{
+  char *fname, *contents;
+  if (bridge_stats_extrainfo)
+    return;
+
+  fname = get_datadir_fname2("stats", "bridge-stats");
+  contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+  if (contents && validate_bridge_stats(contents, now)) {
+    bridge_stats_extrainfo = contents;
+  } else {
+    tor_free(contents);
+  }
+
+  tor_free(fname);
+}
+
+/** Return most recent bridge statistics for inclusion in extra-info
+ * descriptors, or NULL if we don't have recent bridge statistics. */
+const char *
+geoip_get_bridge_stats_extrainfo(time_t now)
+{
+  load_bridge_stats(now);
+  return bridge_stats_extrainfo;
+}
+
+/** Return a new string containing the recent bridge statistics to be returned
+ * to controller clients, or NULL if we don't have any bridge statistics. */
+char *
+geoip_get_bridge_stats_controller(time_t now)
+{
+  return format_bridge_stats_controller(now);
+}
+
+/** Start time of entry stats or 0 if we're not collecting entry
+ * statistics. */
+static time_t start_of_entry_stats_interval;
+
+/** Initialize entry stats. */
+void
+geoip_entry_stats_init(time_t now)
+{
+  start_of_entry_stats_interval = now;
+}
+
+/** Reset counters for entry stats. */
+void
+geoip_reset_entry_stats(time_t now)
+{
+  client_history_clear();
+  start_of_entry_stats_interval = now;
+}
+
+/** Stop collecting entry stats in a way that we can re-start doing so in
+ * geoip_entry_stats_init(). */
+void
+geoip_entry_stats_term(void)
+{
+  geoip_reset_entry_stats(0);
+}
+
+/** Return a newly allocated string containing the entry statistics
+ * until <b>now</b>, or NULL if we're not collecting entry stats. Caller
+ * must ensure start_of_entry_stats_interval lies in the past. */
+char *
+geoip_format_entry_stats(time_t now)
+{
+  char t[ISO_TIME_LEN+1];
+  char *data = NULL;
+  char *result;
+
+  if (!start_of_entry_stats_interval)
+    return NULL; /* Not initialized. */
+
+  tor_assert(now >= start_of_entry_stats_interval);
+
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL);
+  format_iso_time(t, now);
+  tor_asprintf(&result,
+               "entry-stats-end %s (%u s)\n"
+               "entry-ips %s\n",
+               t, (unsigned) (now - start_of_entry_stats_interval),
+               data ? data : "");
+  tor_free(data);
+  return result;
+}
+
+/** If 24 hours have passed since the beginning of the current entry stats
+ * period, write entry stats to $DATADIR/stats/entry-stats (possibly
+ * overwriting an existing file) and reset counters.  Return when we would
+ * next want to write entry stats or 0 if we never want to write. */
+time_t
+geoip_entry_stats_write(time_t now)
+{
+  char *str = NULL;
+
+  if (!start_of_entry_stats_interval)
+    return 0; /* Not initialized. */
+  if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
+    goto done; /* Not ready to write. */
+
+  /* Discard all items in the client history that are too old. */
+  geoip_remove_old_clients(start_of_entry_stats_interval);
+
+  /* Generate history string .*/
+  str = geoip_format_entry_stats(now);
+
+  /* Write entry-stats string to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "entry-stats", str, "entry statistics");
+
+    /* Reset measurement interval start. */
+    geoip_reset_entry_stats(now);
+  }
+
+ done:
+  tor_free(str);
+  return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Helper used to implement GETINFO ip-to-country/... controller command. */
+int
+getinfo_helper_geoip(control_connection_t *control_conn,
+                     const char *question, char **answer,
+                     const char **errmsg)
+{
+  (void)control_conn;
+  if (!strcmpstart(question, "ip-to-country/")) {
+    int c;
+    sa_family_t family;
+    tor_addr_t addr;
+    question += strlen("ip-to-country/");
+
+    if (!strcmp(question, "ipv4-available") ||
+        !strcmp(question, "ipv6-available")) {
+      family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6;
+      const int available = geoip_is_loaded(family);
+      tor_asprintf(answer, "%d", !! available);
+      return 0;
+    }
+
+    family = tor_addr_parse(&addr, question);
+    if (family != AF_INET && family != AF_INET6) {
+      *errmsg = "Invalid address family";
+      return -1;
+    }
+    if (!geoip_is_loaded(family)) {
+      *errmsg = "GeoIP data not loaded";
+      return -1;
+    }
+    if (family == AF_INET)
+      c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
+    else                      /* AF_INET6 */
+      c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
+    *answer = tor_strdup(geoip_get_country_name(c));
+  }
+  return 0;
+}
+
+/** Release all storage held by the GeoIP databases and country list. */
+STATIC void
+clear_geoip_db(void)
+{
+  if (geoip_countries) {
+    SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c));
+    smartlist_free(geoip_countries);
+  }
+
+  strmap_free(country_idxplus1_by_lc_code, NULL);
+  if (geoip_ipv4_entries) {
+    SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent,
+                      tor_free(ent));
+    smartlist_free(geoip_ipv4_entries);
+  }
+  if (geoip_ipv6_entries) {
+    SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent,
+                      tor_free(ent));
+    smartlist_free(geoip_ipv6_entries);
+  }
+  geoip_countries = NULL;
+  country_idxplus1_by_lc_code = NULL;
+  geoip_ipv4_entries = NULL;
+  geoip_ipv6_entries = NULL;
+}
+
+/** Release all storage held in this file. */
+void
+geoip_free_all(void)
+{
+  {
+    clientmap_entry_t **ent, **next, *this;
+    for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
+      this = *ent;
+      next = HT_NEXT_RMV(clientmap, &client_history, ent);
+      clientmap_entry_free(this);
+    }
+    HT_CLEAR(clientmap, &client_history);
+  }
+  {
+    dirreq_map_entry_t **ent, **next, *this;
+    for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
+      this = *ent;
+      next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
+      tor_free(this);
+    }
+    HT_CLEAR(dirreqmap, &dirreq_map);
+  }
+
+  clear_geoip_db();
+  tor_free(bridge_stats_extrainfo);
+
+  memset(geoip_digest, 0, sizeof(geoip_digest));
+  memset(geoip6_digest, 0, sizeof(geoip6_digest));
+}
diff --git a/src/feature/stats/geoip.h b/src/feature/stats/geoip.h
new file mode 100644
index 0000000000..fd19b7560a
--- /dev/null
+++ b/src/feature/stats/geoip.h
@@ -0,0 +1,157 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file geoip.h
+ * \brief Header file for geoip.c.
+ **/
+
+#ifndef TOR_GEOIP_H
+#define TOR_GEOIP_H
+
+#include "lib/testsupport/testsupport.h"
+#include "or/dos.h"
+
+/** Indicates an action that we might be noting geoip statistics on.
+ * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing
+ * the others, we're not.
+ */
+typedef enum {
+  /** We've noticed a connection as a bridge relay or entry guard. */
+  GEOIP_CLIENT_CONNECT = 0,
+  /** We've served a networkstatus consensus as a directory server. */
+  GEOIP_CLIENT_NETWORKSTATUS = 1,
+} geoip_client_action_t;
+/** Indicates either a positive reply or a reason for rejectng a network
+ * status request that will be included in geoip statistics. */
+typedef enum {
+  /** Request is answered successfully. */
+  GEOIP_SUCCESS = 0,
+  /** V3 network status is not signed by a sufficient number of requested
+   * authorities. */
+  GEOIP_REJECT_NOT_ENOUGH_SIGS = 1,
+  /** Requested network status object is unavailable. */
+  GEOIP_REJECT_UNAVAILABLE = 2,
+  /** Requested network status not found. */
+  GEOIP_REJECT_NOT_FOUND = 3,
+  /** Network status has not been modified since If-Modified-Since time. */
+  GEOIP_REJECT_NOT_MODIFIED = 4,
+  /** Directory is busy. */
+  GEOIP_REJECT_BUSY = 5,
+} geoip_ns_response_t;
+#define GEOIP_NS_RESPONSE_NUM 6
+
+/** Directory requests that we are measuring can be either direct or
+ * tunneled. */
+typedef enum {
+  DIRREQ_DIRECT = 0,
+  DIRREQ_TUNNELED = 1,
+} dirreq_type_t;
+
+/** Possible states for either direct or tunneled directory requests that
+ * are relevant for determining network status download times. */
+typedef enum {
+  /** Found that the client requests a network status; applies to both
+   * direct and tunneled requests; initial state of a request that we are
+   * measuring. */
+  DIRREQ_IS_FOR_NETWORK_STATUS = 0,
+  /** Finished writing a network status to the directory connection;
+   * applies to both direct and tunneled requests; completes a direct
+   * request. */
+  DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1,
+  /** END cell sent to circuit that initiated a tunneled request. */
+  DIRREQ_END_CELL_SENT = 2,
+  /** Flushed last cell from queue of the circuit that initiated a
+    * tunneled request to the outbuf of the OR connection. */
+  DIRREQ_CIRC_QUEUE_FLUSHED = 3,
+  /** Flushed last byte from buffer of the channel belonging to the
+    * circuit that initiated a tunneled request; completes a tunneled
+    * request. */
+  DIRREQ_CHANNEL_BUFFER_FLUSHED = 4
+} dirreq_state_t;
+
+#ifdef GEOIP_PRIVATE
+STATIC int geoip_parse_entry(const char *line, sa_family_t family);
+STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr);
+STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr);
+STATIC void clear_geoip_db(void);
+#endif /* defined(GEOIP_PRIVATE) */
+
+/** Entry in a map from IP address to the last time we've seen an incoming
+ * connection from that IP address. Used by bridges only to track which
+ * countries have them blocked, or the DoS mitigation subsystem if enabled. */
+typedef struct clientmap_entry_t {
+  HT_ENTRY(clientmap_entry_t) node;
+  tor_addr_t addr;
+  /* Name of pluggable transport used by this client. NULL if no
+     pluggable transport was used. */
+  char *transport_name;
+
+  /** Time when we last saw this IP address, in MINUTES since the epoch.
+   *
+   * (This will run out of space around 4011 CE.  If Tor is still in use around
+   * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */
+  unsigned int last_seen_in_minutes:30;
+  unsigned int action:2;
+
+  /* This object is used to keep some statistics per client address for the
+   * DoS mitigation subsystem. */
+  dos_client_stats_t dos_stats;
+} clientmap_entry_t;
+
+int should_record_bridge_info(const or_options_t *options);
+int geoip_load_file(sa_family_t family, const char *filename);
+MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
+MOCK_DECL(int, geoip_get_n_countries, (void));
+const char *geoip_get_country_name(country_t num);
+MOCK_DECL(int, geoip_is_loaded, (sa_family_t family));
+const char *geoip_db_digest(sa_family_t family);
+MOCK_DECL(country_t, geoip_get_country, (const char *countrycode));
+
+void geoip_note_client_seen(geoip_client_action_t action,
+                            const tor_addr_t *addr, const char *transport_name,
+                            time_t now);
+void geoip_remove_old_clients(time_t cutoff);
+clientmap_entry_t *geoip_lookup_client(const tor_addr_t *addr,
+                                       const char *transport_name,
+                                       geoip_client_action_t action);
+size_t geoip_client_cache_total_allocation(void);
+size_t geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes);
+
+void geoip_note_ns_response(geoip_ns_response_t response);
+char *geoip_get_transport_history(void);
+int geoip_get_client_history(geoip_client_action_t action,
+                             char **country_str, char **ipver_str);
+char *geoip_get_request_history(void);
+int getinfo_helper_geoip(control_connection_t *control_conn,
+                         const char *question, char **answer,
+                         const char **errmsg);
+void geoip_free_all(void);
+
+void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
+                        dirreq_type_t type);
+void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
+                               dirreq_state_t new_state);
+
+void geoip_dirreq_stats_init(time_t now);
+void geoip_reset_dirreq_stats(time_t now);
+char *geoip_format_dirreq_stats(time_t now);
+time_t geoip_dirreq_stats_write(time_t now);
+void geoip_dirreq_stats_term(void);
+void geoip_entry_stats_init(time_t now);
+time_t geoip_entry_stats_write(time_t now);
+void geoip_entry_stats_term(void);
+void geoip_reset_entry_stats(time_t now);
+char *geoip_format_entry_stats(time_t now);
+void geoip_bridge_stats_init(time_t now);
+char *geoip_format_bridge_stats(time_t now);
+time_t geoip_bridge_stats_write(time_t now);
+void geoip_bridge_stats_term(void);
+const char *geoip_get_bridge_stats_extrainfo(time_t);
+char *geoip_get_bridge_stats_controller(time_t);
+char *format_client_stats_heartbeat(time_t now);
+
+#endif /* !defined(TOR_GEOIP_H) */
diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c
new file mode 100644
index 0000000000..6607c25964
--- /dev/null
+++ b/src/feature/stats/rephist.c
@@ -0,0 +1,3220 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rephist.c
+ * \brief Basic history and performance-tracking functionality.
+ *
+ * Basic history and performance-tracking functionality to remember
+ *    which servers have worked in the past, how much bandwidth we've
+ *    been using, which ports we tend to want, and so on; further,
+ *    exit port statistics, cell statistics, and connection statistics.
+ *
+ * The history and information tracked in this module could sensibly be
+ * divided into several categories:
+ *
+ * <ul><li>Statistics used by authorities to remember the uptime and
+ * stability information about various relays, including "uptime",
+ * "weighted fractional uptime" and "mean time between failures".
+ *
+ * <li>Bandwidth usage history, used by relays to self-report how much
+ * bandwidth they've used for different purposes over last day or so,
+ * in order to generate the {dirreq-,}{read,write}-history lines in
+ * that they publish.
+ *
+ * <li>Predicted ports, used by clients to remember how long it's been
+ * since they opened an exit connection to each given target
+ * port. Clients use this information in order to try to keep circuits
+ * open to exit nodes that can connect to the ports that they care
+ * about.  (The predicted ports mechanism also handles predicted circuit
+ * usage that _isn't_ port-specific, such as resolves, internal circuits,
+ * and so on.)
+ *
+ * <li>Public key operation counters, for tracking how many times we've
+ * done each public key operation.  (This is unmaintained and we should
+ * remove it.)
+ *
+ * <li>Exit statistics by port, used by exits to keep track of the
+ * number of streams and bytes they've served at each exit port, so they
+ * can generate their exit-kibibytes-{read,written} and
+ * exit-streams-opened statistics.
+ *
+ * <li>Circuit stats, used by relays instances to tract circuit
+ * queue fullness and delay over time, and generate cell-processed-cells,
+ * cell-queued-cells, cell-time-in-queue, and cell-circuits-per-decile
+ * statistics.
+ *
+ * <li>Descriptor serving statistics, used by directory caches to track
+ * how many descriptors they've served.
+ *
+ * <li>Connection statistics, used by relays to track one-way and
+ * bidirectional connections.
+ *
+ * <li>Onion handshake statistics, used by relays to count how many
+ * TAP and ntor handshakes they've handled.
+ *
+ * <li>Hidden service statistics, used by relays to count rendezvous
+ * traffic and HSDir-stored descriptors.
+ *
+ * <li>Link protocol statistics, used by relays to count how many times
+ * each link protocol has been used.
+ *
+ * </ul>
+ *
+ * The entry points for this module are scattered throughout the
+ * codebase.  Sending data, receiving data, connecting to a relay,
+ * losing a connection to a relay, and so on can all trigger a change in
+ * our current stats.  Relays also invoke this module in order to
+ * extract their statistics when building routerinfo and extrainfo
+ * objects in router.c.
+ *
+ * TODO: This module should be broken up.
+ *
+ * (The "rephist" name originally stood for "reputation and history". )
+ **/
+
+#include "or/or.h"
+#include "or/circuitlist.h"
+#include "or/circuituse.h"
+#include "or/config.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "or/networkstatus.h"
+#include "or/nodelist.h"
+#include "or/rephist.h"
+#include "or/router.h"
+#include "or/routerlist.h"
+#include "ht.h"
+#include "or/channelpadding.h"
+#include "or/connection_or.h"
+#include "or/statefile.h"
+
+#include "or/networkstatus_st.h"
+#include "or/or_circuit_st.h"
+#include "or/or_state_st.h"
+
+#include "lib/container/bloomfilt.h"
+#include "lib/container/order.h"
+#include "lib/math/fp.h"
+#include "lib/math/laplace.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+static void bw_arrays_init(void);
+static void predicted_ports_alloc(void);
+
+/** Total number of bytes currently allocated in fields used by rephist.c. */
+uint64_t rephist_total_alloc=0;
+/** Number of or_history_t objects currently allocated. */
+uint32_t rephist_total_num=0;
+
+/** If the total weighted run count of all runs for a router ever falls
+ * below this amount, the router can be treated as having 0 MTBF. */
+#define STABILITY_EPSILON   0.0001
+/** Value by which to discount all old intervals for MTBF purposes.  This
+ * is compounded every STABILITY_INTERVAL. */
+#define STABILITY_ALPHA     0.95
+/** Interval at which to discount all old intervals for MTBF purposes. */
+#define STABILITY_INTERVAL  (12*60*60)
+/* (This combination of ALPHA, INTERVAL, and EPSILON makes it so that an
+ * interval that just ended counts twice as much as one that ended a week ago,
+ * 20X as much as one that ended a month ago, and routers that have had no
+ * uptime data for about half a year will get forgotten.) */
+
+/** History of an OR. */
+typedef struct or_history_t {
+  /** When did we start tracking this OR? */
+  time_t since;
+  /** When did we most recently note a change to this OR? */
+  time_t changed;
+
+  /** The address at which we most recently connected to this OR
+   * successfully. */
+  tor_addr_t last_reached_addr;
+
+  /** The port at which we most recently connected to this OR successfully */
+  uint16_t last_reached_port;
+
+  /* === For MTBF tracking: */
+  /** Weighted sum total of all times that this router has been online.
+   */
+  unsigned long weighted_run_length;
+  /** If the router is now online (according to stability-checking rules),
+   * when did it come online? */
+  time_t start_of_run;
+  /** Sum of weights for runs in weighted_run_length. */
+  double total_run_weights;
+  /* === For fractional uptime tracking: */
+  time_t start_of_downtime;
+  unsigned long weighted_uptime;
+  unsigned long total_weighted_time;
+} or_history_t;
+
+/**
+ * This structure holds accounting needed to calculate the padding overhead.
+ */
+typedef struct padding_counts_t {
+  /** Total number of cells we have received, including padding */
+  uint64_t read_cell_count;
+  /** Total number of cells we have sent, including padding */
+  uint64_t write_cell_count;
+  /** Total number of CELL_PADDING cells we have received */
+  uint64_t read_pad_cell_count;
+  /** Total number of CELL_PADDING cells we have sent */
+  uint64_t write_pad_cell_count;
+  /** Total number of read cells on padding-enabled conns */
+  uint64_t enabled_read_cell_count;
+  /** Total number of sent cells on padding-enabled conns */
+  uint64_t enabled_write_cell_count;
+  /** Total number of read CELL_PADDING cells on padding-enabled cons */
+  uint64_t enabled_read_pad_cell_count;
+  /** Total number of sent CELL_PADDING cells on padding-enabled cons */
+  uint64_t enabled_write_pad_cell_count;
+  /** Total number of RELAY_DROP cells we have received */
+  uint64_t read_drop_cell_count;
+  /** Total number of RELAY_DROP cells we have sent */
+  uint64_t write_drop_cell_count;
+  /** The maximum number of padding timers we've seen in 24 hours */
+  uint64_t maximum_chanpad_timers;
+  /** When did we first copy padding_current into padding_published? */
+  char first_published_at[ISO_TIME_LEN+1];
+} padding_counts_t;
+
+/** Holds the current values of our padding statistics.
+ * It is not published until it is transferred to padding_published. */
+static padding_counts_t padding_current;
+
+/** Remains fixed for a 24 hour period, and then is replaced
+ * by a redacted copy of padding_current */
+static padding_counts_t padding_published;
+
+/** When did we last multiply all routers' weighted_run_length and
+ * total_run_weights by STABILITY_ALPHA? */
+static time_t stability_last_downrated = 0;
+
+/**  */
+static time_t started_tracking_stability = 0;
+
+/** Map from hex OR identity digest to or_history_t. */
+static digestmap_t *history_map = NULL;
+
+/** Return the or_history_t for the OR with identity digest <b>id</b>,
+ * creating it if necessary. */
+static or_history_t *
+get_or_history(const char* id)
+{
+  or_history_t *hist;
+
+  if (tor_digest_is_zero(id))
+    return NULL;
+
+  hist = digestmap_get(history_map, id);
+  if (!hist) {
+    hist = tor_malloc_zero(sizeof(or_history_t));
+    rephist_total_alloc += sizeof(or_history_t);
+    rephist_total_num++;
+    hist->since = hist->changed = time(NULL);
+    tor_addr_make_unspec(&hist->last_reached_addr);
+    digestmap_set(history_map, id, hist);
+  }
+  return hist;
+}
+
+/** Helper: free storage held by a single OR history entry. */
+static void
+free_or_history(void *_hist)
+{
+  or_history_t *hist = _hist;
+  rephist_total_alloc -= sizeof(or_history_t);
+  rephist_total_num--;
+  tor_free(hist);
+}
+
+/** Initialize the static data structures for tracking history. */
+void
+rep_hist_init(void)
+{
+  history_map = digestmap_new();
+  bw_arrays_init();
+  predicted_ports_alloc();
+}
+
+/** We have just decided that this router with identity digest <b>id</b> is
+ * reachable, meaning we will give it a "Running" flag for the next while. */
+void
+rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr,
+                               const uint16_t at_port, time_t when)
+{
+  or_history_t *hist = get_or_history(id);
+  int was_in_run = 1;
+  char tbuf[ISO_TIME_LEN+1];
+  int addr_changed, port_changed;
+
+  tor_assert(hist);
+  tor_assert((!at_addr && !at_port) || (at_addr && at_port));
+
+  addr_changed = at_addr && !tor_addr_is_null(&hist->last_reached_addr) &&
+    tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0;
+  port_changed = at_port && hist->last_reached_port &&
+                 at_port != hist->last_reached_port;
+
+  if (!started_tracking_stability)
+    started_tracking_stability = time(NULL);
+  if (!hist->start_of_run) {
+    hist->start_of_run = when;
+    was_in_run = 0;
+  }
+  if (hist->start_of_downtime) {
+    long down_length;
+
+    format_local_iso_time(tbuf, hist->start_of_downtime);
+    log_info(LD_HIST, "Router %s is now Running; it had been down since %s.",
+             hex_str(id, DIGEST_LEN), tbuf);
+    if (was_in_run)
+      log_info(LD_HIST, "  (Paradoxically, it was already Running too.)");
+
+    down_length = when - hist->start_of_downtime;
+    hist->total_weighted_time += down_length;
+    hist->start_of_downtime = 0;
+  } else if (addr_changed || port_changed) {
+    /* If we're reachable, but the address changed, treat this as some
+     * downtime. */
+    int penalty = get_options()->TestingTorNetwork ? 240 : 3600;
+    networkstatus_t *ns;
+
+    if ((ns = networkstatus_get_latest_consensus())) {
+      int fresh_interval = (int)(ns->fresh_until - ns->valid_after);
+      int live_interval = (int)(ns->valid_until - ns->valid_after);
+      /* on average, a descriptor addr change takes .5 intervals to make it
+       * into a consensus, and half a liveness period to make it to
+       * clients. */
+      penalty = (int)(fresh_interval + live_interval) / 2;
+    }
+    format_local_iso_time(tbuf, hist->start_of_run);
+    log_info(LD_HIST,"Router %s still seems Running, but its address appears "
+             "to have changed since the last time it was reachable.  I'm "
+             "going to treat it as having been down for %d seconds",
+             hex_str(id, DIGEST_LEN), penalty);
+    rep_hist_note_router_unreachable(id, when-penalty);
+    rep_hist_note_router_reachable(id, NULL, 0, when);
+  } else {
+    format_local_iso_time(tbuf, hist->start_of_run);
+    if (was_in_run)
+      log_debug(LD_HIST, "Router %s is still Running; it has been Running "
+                "since %s", hex_str(id, DIGEST_LEN), tbuf);
+    else
+      log_info(LD_HIST,"Router %s is now Running; it was previously untracked",
+               hex_str(id, DIGEST_LEN));
+  }
+  if (at_addr)
+    tor_addr_copy(&hist->last_reached_addr, at_addr);
+  if (at_port)
+    hist->last_reached_port = at_port;
+}
+
+/** We have just decided that this router is unreachable, meaning
+ * we are taking away its "Running" flag. */
+void
+rep_hist_note_router_unreachable(const char *id, time_t when)
+{
+  or_history_t *hist = get_or_history(id);
+  char tbuf[ISO_TIME_LEN+1];
+  int was_running = 0;
+  if (!started_tracking_stability)
+    started_tracking_stability = time(NULL);
+
+  tor_assert(hist);
+  if (hist->start_of_run) {
+    /*XXXX We could treat failed connections differently from failed
+     * connect attempts. */
+    long run_length = when - hist->start_of_run;
+    format_local_iso_time(tbuf, hist->start_of_run);
+
+    hist->total_run_weights += 1.0;
+    hist->start_of_run = 0;
+    if (run_length < 0) {
+      unsigned long penalty = -run_length;
+#define SUBTRACT_CLAMPED(var, penalty) \
+      do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0)
+
+      SUBTRACT_CLAMPED(hist->weighted_run_length, penalty);
+      SUBTRACT_CLAMPED(hist->weighted_uptime, penalty);
+    } else {
+      hist->weighted_run_length += run_length;
+      hist->weighted_uptime += run_length;
+      hist->total_weighted_time += run_length;
+    }
+    was_running = 1;
+    log_info(LD_HIST, "Router %s is now non-Running: it had previously been "
+             "Running since %s.  Its total weighted uptime is %lu/%lu.",
+             hex_str(id, DIGEST_LEN), tbuf, hist->weighted_uptime,
+             hist->total_weighted_time);
+  }
+  if (!hist->start_of_downtime) {
+    hist->start_of_downtime = when;
+
+    if (!was_running)
+      log_info(LD_HIST, "Router %s is now non-Running; it was previously "
+               "untracked.", hex_str(id, DIGEST_LEN));
+  } else {
+    if (!was_running) {
+      format_local_iso_time(tbuf, hist->start_of_downtime);
+
+      log_info(LD_HIST, "Router %s is still non-Running; it has been "
+               "non-Running since %s.", hex_str(id, DIGEST_LEN), tbuf);
+    }
+  }
+}
+
+/** Mark a router with ID <b>id</b> as non-Running, and retroactively declare
+ * that it has never been running: give it no stability and no WFU. */
+void
+rep_hist_make_router_pessimal(const char *id, time_t when)
+{
+  or_history_t *hist = get_or_history(id);
+  tor_assert(hist);
+
+  rep_hist_note_router_unreachable(id, when);
+
+  hist->weighted_run_length = 0;
+  hist->weighted_uptime = 0;
+}
+
+/** Helper: Discount all old MTBF data, if it is time to do so.  Return
+ * the time at which we should next discount MTBF data. */
+time_t
+rep_hist_downrate_old_runs(time_t now)
+{
+  digestmap_iter_t *orhist_it;
+  const char *digest1;
+  or_history_t *hist;
+  void *hist_p;
+  double alpha = 1.0;
+
+  if (!history_map)
+    history_map = digestmap_new();
+  if (!stability_last_downrated)
+    stability_last_downrated = now;
+  if (stability_last_downrated + STABILITY_INTERVAL > now)
+    return stability_last_downrated + STABILITY_INTERVAL;
+
+  /* Okay, we should downrate the data.  By how much? */
+  while (stability_last_downrated + STABILITY_INTERVAL < now) {
+    stability_last_downrated += STABILITY_INTERVAL;
+    alpha *= STABILITY_ALPHA;
+  }
+
+  log_info(LD_HIST, "Discounting all old stability info by a factor of %f",
+           alpha);
+
+  /* Multiply every w_r_l, t_r_w pair by alpha. */
+  for (orhist_it = digestmap_iter_init(history_map);
+       !digestmap_iter_done(orhist_it);
+       orhist_it = digestmap_iter_next(history_map,orhist_it)) {
+    digestmap_iter_get(orhist_it, &digest1, &hist_p);
+    hist = hist_p;
+
+    hist->weighted_run_length =
+      (unsigned long)(hist->weighted_run_length * alpha);
+    hist->total_run_weights *= alpha;
+
+    hist->weighted_uptime = (unsigned long)(hist->weighted_uptime * alpha);
+    hist->total_weighted_time = (unsigned long)
+      (hist->total_weighted_time * alpha);
+  }
+
+  return stability_last_downrated + STABILITY_INTERVAL;
+}
+
+/** Helper: Return the weighted MTBF of the router with history <b>hist</b>. */
+static double
+get_stability(or_history_t *hist, time_t when)
+{
+  long total = hist->weighted_run_length;
+  double total_weights = hist->total_run_weights;
+
+  if (hist->start_of_run) {
+    /* We're currently in a run.  Let total and total_weights hold the values
+     * they would hold if the current run were to end now. */
+    total += (when-hist->start_of_run);
+    total_weights += 1.0;
+  }
+  if (total_weights < STABILITY_EPSILON) {
+    /* Round down to zero, and avoid divide-by-zero. */
+    return 0.0;
+  }
+
+  return total / total_weights;
+}
+
+/** Return the total amount of time we've been observing, with each run of
+ * time downrated by the appropriate factor. */
+static long
+get_total_weighted_time(or_history_t *hist, time_t when)
+{
+  long total = hist->total_weighted_time;
+  if (hist->start_of_run) {
+    total += (when - hist->start_of_run);
+  } else if (hist->start_of_downtime) {
+    total += (when - hist->start_of_downtime);
+  }
+  return total;
+}
+
+/** Helper: Return the weighted percent-of-time-online of the router with
+ * history <b>hist</b>. */
+static double
+get_weighted_fractional_uptime(or_history_t *hist, time_t when)
+{
+  long total = hist->total_weighted_time;
+  long up = hist->weighted_uptime;
+
+  if (hist->start_of_run) {
+    long run_length = (when - hist->start_of_run);
+    up += run_length;
+    total += run_length;
+  } else if (hist->start_of_downtime) {
+    total += (when - hist->start_of_downtime);
+  }
+
+  if (!total) {
+    /* Avoid calling anybody's uptime infinity (which should be impossible if
+     * the code is working), or NaN (which can happen for any router we haven't
+     * observed up or down yet). */
+    return 0.0;
+  }
+
+  return ((double) up) / total;
+}
+
+/** Return how long the router whose identity digest is <b>id</b> has
+ *  been reachable. Return 0 if the router is unknown or currently deemed
+ *  unreachable. */
+long
+rep_hist_get_uptime(const char *id, time_t when)
+{
+  or_history_t *hist = get_or_history(id);
+  if (!hist)
+    return 0;
+  if (!hist->start_of_run || when < hist->start_of_run)
+    return 0;
+  return when - hist->start_of_run;
+}
+
+/** Return an estimated MTBF for the router whose identity digest is
+ * <b>id</b>. Return 0 if the router is unknown. */
+double
+rep_hist_get_stability(const char *id, time_t when)
+{
+  or_history_t *hist = get_or_history(id);
+  if (!hist)
+    return 0.0;
+
+  return get_stability(hist, when);
+}
+
+/** Return an estimated percent-of-time-online for the router whose identity
+ * digest is <b>id</b>. Return 0 if the router is unknown. */
+double
+rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
+{
+  or_history_t *hist = get_or_history(id);
+  if (!hist)
+    return 0.0;
+
+  return get_weighted_fractional_uptime(hist, when);
+}
+
+/** Return a number representing how long we've known about the router whose
+ * digest is <b>id</b>. Return 0 if the router is unknown.
+ *
+ * Be careful: this measure increases monotonically as we know the router for
+ * longer and longer, but it doesn't increase linearly.
+ */
+long
+rep_hist_get_weighted_time_known(const char *id, time_t when)
+{
+  or_history_t *hist = get_or_history(id);
+  if (!hist)
+    return 0;
+
+  return get_total_weighted_time(hist, when);
+}
+
+/** Return true if we've been measuring MTBFs for long enough to
+ * pronounce on Stability. */
+int
+rep_hist_have_measured_enough_stability(void)
+{
+  /* XXXX++ This doesn't do so well when we change our opinion
+   * as to whether we're tracking router stability. */
+  return started_tracking_stability < time(NULL) - 4*60*60;
+}
+
+/** Log all the reliability data we have remembered, with the chosen
+ * severity.
+ */
+void
+rep_hist_dump_stats(time_t now, int severity)
+{
+  digestmap_iter_t *orhist_it;
+  const char *name1, *digest1;
+  char hexdigest1[HEX_DIGEST_LEN+1];
+  or_history_t *or_history;
+  void *or_history_p;
+  const node_t *node;
+
+  rep_history_clean(now - get_options()->RephistTrackTime);
+
+  tor_log(severity, LD_HIST, "--------------- Dumping history information:");
+
+  for (orhist_it = digestmap_iter_init(history_map);
+       !digestmap_iter_done(orhist_it);
+       orhist_it = digestmap_iter_next(history_map,orhist_it)) {
+    double s;
+    long stability;
+    digestmap_iter_get(orhist_it, &digest1, &or_history_p);
+    or_history = (or_history_t*) or_history_p;
+
+    if ((node = node_get_by_id(digest1)) && node_get_nickname(node))
+      name1 = node_get_nickname(node);
+    else
+      name1 = "(unknown)";
+    base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN);
+    s = get_stability(or_history, now);
+    stability = (long)s;
+    tor_log(severity, LD_HIST,
+        "OR %s [%s]: wmtbf %lu:%02lu:%02lu",
+        name1, hexdigest1,
+        stability/3600, (stability/60)%60, stability%60);
+  }
+}
+
+/** Remove history info for routers/links that haven't changed since
+ * <b>before</b>.
+ */
+void
+rep_history_clean(time_t before)
+{
+  int authority = authdir_mode(get_options());
+  or_history_t *or_history;
+  void *or_history_p;
+  digestmap_iter_t *orhist_it;
+  const char *d1;
+
+  orhist_it = digestmap_iter_init(history_map);
+  while (!digestmap_iter_done(orhist_it)) {
+    int should_remove;
+    digestmap_iter_get(orhist_it, &d1, &or_history_p);
+    or_history = or_history_p;
+
+    should_remove = authority ?
+                       (or_history->total_run_weights < STABILITY_EPSILON &&
+                          !or_history->start_of_run)
+                       : (or_history->changed < before);
+    if (should_remove) {
+      orhist_it = digestmap_iter_next_rmv(history_map, orhist_it);
+      free_or_history(or_history);
+      continue;
+    }
+    orhist_it = digestmap_iter_next(history_map, orhist_it);
+  }
+}
+
+/** Write MTBF data to disk. Return 0 on success, negative on failure.
+ *
+ * If <b>missing_means_down</b>, then if we're about to write an entry
+ * that is still considered up but isn't in our routerlist, consider it
+ * to be down. */
+int
+rep_hist_record_mtbf_data(time_t now, int missing_means_down)
+{
+  char time_buf[ISO_TIME_LEN+1];
+
+  digestmap_iter_t *orhist_it;
+  const char *digest;
+  void *or_history_p;
+  or_history_t *hist;
+  open_file_t *open_file = NULL;
+  FILE *f;
+
+  {
+    char *filename = get_datadir_fname("router-stability");
+    f = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE|O_TEXT, 0600,
+                                    &open_file);
+    tor_free(filename);
+    if (!f)
+      return -1;
+  }
+
+  /* File format is:
+   *   FormatLine *KeywordLine Data
+   *
+   *   FormatLine = "format 1" NL
+   *   KeywordLine = Keyword SP Arguments NL
+   *   Data = "data" NL *RouterMTBFLine "." NL
+   *   RouterMTBFLine = Fingerprint SP WeightedRunLen SP
+   *           TotalRunWeights [SP S=StartRunTime] NL
+   */
+#define PUT(s) STMT_BEGIN if (fputs((s),f)<0) goto err; STMT_END
+#define PRINTF(args) STMT_BEGIN if (fprintf args <0) goto err; STMT_END
+
+  PUT("format 2\n");
+
+  format_iso_time(time_buf, time(NULL));
+  PRINTF((f, "stored-at %s\n", time_buf));
+
+  if (started_tracking_stability) {
+    format_iso_time(time_buf, started_tracking_stability);
+    PRINTF((f, "tracked-since %s\n", time_buf));
+  }
+  if (stability_last_downrated) {
+    format_iso_time(time_buf, stability_last_downrated);
+    PRINTF((f, "last-downrated %s\n", time_buf));
+  }
+
+  PUT("data\n");
+
+  /* XXX Nick: now bridge auths record this for all routers too.
+   * Should we make them record it only for bridge routers? -RD
+   * Not for 0.2.0. -NM */
+  for (orhist_it = digestmap_iter_init(history_map);
+       !digestmap_iter_done(orhist_it);
+       orhist_it = digestmap_iter_next(history_map,orhist_it)) {
+    char dbuf[HEX_DIGEST_LEN+1];
+    const char *t = NULL;
+    digestmap_iter_get(orhist_it, &digest, &or_history_p);
+    hist = (or_history_t*) or_history_p;
+
+    base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
+
+    if (missing_means_down && hist->start_of_run &&
+        !connection_or_digest_is_known_relay(digest)) {
+      /* We think this relay is running, but it's not listed in our
+       * consensus. Somehow it fell out without telling us it went
+       * down. Complain and also correct it. */
+      log_info(LD_HIST,
+               "Relay '%s' is listed as up in rephist, but it's not in "
+               "our routerlist. Correcting.", dbuf);
+      rep_hist_note_router_unreachable(digest, now);
+    }
+
+    PRINTF((f, "R %s\n", dbuf));
+    if (hist->start_of_run > 0) {
+      format_iso_time(time_buf, hist->start_of_run);
+      t = time_buf;
+    }
+    PRINTF((f, "+MTBF %lu %.5f%s%s\n",
+            hist->weighted_run_length, hist->total_run_weights,
+            t ? " S=" : "", t ? t : ""));
+    t = NULL;
+    if (hist->start_of_downtime > 0) {
+      format_iso_time(time_buf, hist->start_of_downtime);
+      t = time_buf;
+    }
+    PRINTF((f, "+WFU %lu %lu%s%s\n",
+            hist->weighted_uptime, hist->total_weighted_time,
+            t ? " S=" : "", t ? t : ""));
+  }
+
+  PUT(".\n");
+
+#undef PUT
+#undef PRINTF
+
+  return finish_writing_to_file(open_file);
+ err:
+  abort_writing_to_file(open_file);
+  return -1;
+}
+
+/** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and
+ * such that no line sl[k] with i <= k < j starts with "R ".  Return -1 if no
+ * such line exists. */
+static int
+find_next_with(smartlist_t *sl, int i, const char *prefix)
+{
+  for ( ; i < smartlist_len(sl); ++i) {
+    const char *line = smartlist_get(sl, i);
+    if (!strcmpstart(line, prefix))
+      return i;
+    if (!strcmpstart(line, "R "))
+      return -1;
+  }
+  return -1;
+}
+
+/** How many bad times has parse_possibly_bad_iso_time() parsed? */
+static int n_bogus_times = 0;
+/** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but
+ * round any pre-1970 date to Jan 1, 1970. */
+static int
+parse_possibly_bad_iso_time(const char *s, time_t *time_out)
+{
+  int year;
+  char b[5];
+  strlcpy(b, s, sizeof(b));
+  b[4] = '\0';
+  year = (int)tor_parse_long(b, 10, 0, INT_MAX, NULL, NULL);
+  if (year < 1970) {
+    *time_out = 0;
+    ++n_bogus_times;
+    return 0;
+  } else
+    return parse_iso_time(s, time_out);
+}
+
+/** We've read a time <b>t</b> from a file stored at <b>stored_at</b>, which
+ * says we started measuring at <b>started_measuring</b>.  Return a new number
+ * that's about as much before <b>now</b> as <b>t</b> was before
+ * <b>stored_at</b>.
+ */
+static inline time_t
+correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
+{
+  if (t < started_measuring - 24*60*60*365)
+    return 0;
+  else if (t < started_measuring)
+    return started_measuring;
+  else if (t > stored_at)
+    return 0;
+  else {
+    long run_length = stored_at - t;
+    t = (time_t)(now - run_length);
+    if (t < started_measuring)
+      t = started_measuring;
+    return t;
+  }
+}
+
+/** Load MTBF data from disk.  Returns 0 on success or recoverable error, -1
+ * on failure. */
+int
+rep_hist_load_mtbf_data(time_t now)
+{
+  /* XXXX won't handle being called while history is already populated. */
+  smartlist_t *lines;
+  const char *line = NULL;
+  int r=0, i;
+  time_t last_downrated = 0, stored_at = 0, tracked_since = 0;
+  time_t latest_possible_start = now;
+  long format = -1;
+
+  {
+    char *filename = get_datadir_fname("router-stability");
+    char *d = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+    tor_free(filename);
+    if (!d)
+      return -1;
+    lines = smartlist_new();
+    smartlist_split_string(lines, d, "\n", SPLIT_SKIP_SPACE, 0);
+    tor_free(d);
+  }
+
+  {
+    const char *firstline;
+    if (smartlist_len(lines)>4) {
+      firstline = smartlist_get(lines, 0);
+      if (!strcmpstart(firstline, "format "))
+        format = tor_parse_long(firstline+strlen("format "),
+                                10, -1, LONG_MAX, NULL, NULL);
+    }
+  }
+  if (format != 1 && format != 2) {
+    log_warn(LD_HIST,
+             "Unrecognized format in mtbf history file. Skipping.");
+    goto err;
+  }
+  for (i = 1; i < smartlist_len(lines); ++i) {
+    line = smartlist_get(lines, i);
+    if (!strcmp(line, "data"))
+      break;
+    if (!strcmpstart(line, "last-downrated ")) {
+      if (parse_iso_time(line+strlen("last-downrated "), &last_downrated)<0)
+        log_warn(LD_HIST,"Couldn't parse downrate time in mtbf "
+                 "history file.");
+    }
+    if (!strcmpstart(line, "stored-at ")) {
+      if (parse_iso_time(line+strlen("stored-at "), &stored_at)<0)
+        log_warn(LD_HIST,"Couldn't parse stored time in mtbf "
+                 "history file.");
+    }
+    if (!strcmpstart(line, "tracked-since ")) {
+      if (parse_iso_time(line+strlen("tracked-since "), &tracked_since)<0)
+        log_warn(LD_HIST,"Couldn't parse started-tracking time in mtbf "
+                 "history file.");
+    }
+  }
+  if (last_downrated > now)
+    last_downrated = now;
+  if (tracked_since > now)
+    tracked_since = now;
+
+  if (!stored_at) {
+    log_warn(LD_HIST, "No stored time recorded.");
+    goto err;
+  }
+
+  if (line && !strcmp(line, "data"))
+    ++i;
+
+  n_bogus_times = 0;
+
+  for (; i < smartlist_len(lines); ++i) {
+    char digest[DIGEST_LEN];
+    char hexbuf[HEX_DIGEST_LEN+1];
+    char mtbf_timebuf[ISO_TIME_LEN+1];
+    char wfu_timebuf[ISO_TIME_LEN+1];
+    time_t start_of_run = 0;
+    time_t start_of_downtime = 0;
+    int have_mtbf = 0, have_wfu = 0;
+    long wrl = 0;
+    double trw = 0;
+    long wt_uptime = 0, total_wt_time = 0;
+    int n;
+    or_history_t *hist;
+    line = smartlist_get(lines, i);
+    if (!strcmp(line, "."))
+      break;
+
+    mtbf_timebuf[0] = '\0';
+    wfu_timebuf[0] = '\0';
+
+    if (format == 1) {
+      n = tor_sscanf(line, "%40s %ld %lf S=%10s %8s",
+                 hexbuf, &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
+      if (n != 3 && n != 5) {
+        log_warn(LD_HIST, "Couldn't scan line %s", escaped(line));
+        continue;
+      }
+      have_mtbf = 1;
+    } else {
+      // format == 2.
+      int mtbf_idx, wfu_idx;
+      if (strcmpstart(line, "R ") || strlen(line) < 2+HEX_DIGEST_LEN)
+        continue;
+      strlcpy(hexbuf, line+2, sizeof(hexbuf));
+      mtbf_idx = find_next_with(lines, i+1, "+MTBF ");
+      wfu_idx = find_next_with(lines, i+1, "+WFU ");
+      if (mtbf_idx >= 0) {
+        const char *mtbfline = smartlist_get(lines, mtbf_idx);
+        n = tor_sscanf(mtbfline, "+MTBF %lu %lf S=%10s %8s",
+                   &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
+        if (n == 2 || n == 4) {
+          have_mtbf = 1;
+        } else {
+          log_warn(LD_HIST, "Couldn't scan +MTBF line %s",
+                   escaped(mtbfline));
+        }
+      }
+      if (wfu_idx >= 0) {
+        const char *wfuline = smartlist_get(lines, wfu_idx);
+        n = tor_sscanf(wfuline, "+WFU %lu %lu S=%10s %8s",
+                   &wt_uptime, &total_wt_time,
+                   wfu_timebuf, wfu_timebuf+11);
+        if (n == 2 || n == 4) {
+          have_wfu = 1;
+        } else {
+          log_warn(LD_HIST, "Couldn't scan +WFU line %s", escaped(wfuline));
+        }
+      }
+      if (wfu_idx > i)
+        i = wfu_idx;
+      if (mtbf_idx > i)
+        i = mtbf_idx;
+    }
+    if (base16_decode(digest, DIGEST_LEN,
+                      hexbuf, HEX_DIGEST_LEN) != DIGEST_LEN) {
+      log_warn(LD_HIST, "Couldn't hex string %s", escaped(hexbuf));
+      continue;
+    }
+    hist = get_or_history(digest);
+    if (!hist)
+      continue;
+
+    if (have_mtbf) {
+      if (mtbf_timebuf[0]) {
+        mtbf_timebuf[10] = ' ';
+        if (parse_possibly_bad_iso_time(mtbf_timebuf, &start_of_run)<0)
+          log_warn(LD_HIST, "Couldn't parse time %s",
+                   escaped(mtbf_timebuf));
+      }
+      hist->start_of_run = correct_time(start_of_run, now, stored_at,
+                                        tracked_since);
+      if (hist->start_of_run < latest_possible_start + wrl)
+        latest_possible_start = (time_t)(hist->start_of_run - wrl);
+
+      hist->weighted_run_length = wrl;
+      hist->total_run_weights = trw;
+    }
+    if (have_wfu) {
+      if (wfu_timebuf[0]) {
+        wfu_timebuf[10] = ' ';
+        if (parse_possibly_bad_iso_time(wfu_timebuf, &start_of_downtime)<0)
+          log_warn(LD_HIST, "Couldn't parse time %s", escaped(wfu_timebuf));
+      }
+    }
+    hist->start_of_downtime = correct_time(start_of_downtime, now, stored_at,
+                                           tracked_since);
+    hist->weighted_uptime = wt_uptime;
+    hist->total_weighted_time = total_wt_time;
+  }
+  if (strcmp(line, "."))
+    log_warn(LD_HIST, "Truncated MTBF file.");
+
+  if (tracked_since < 86400*365) /* Recover from insanely early value. */
+    tracked_since = latest_possible_start;
+
+  stability_last_downrated = last_downrated;
+  started_tracking_stability = tracked_since;
+
+  goto done;
+ err:
+  r = -1;
+ done:
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+  return r;
+}
+
+/** For how many seconds do we keep track of individual per-second bandwidth
+ * totals? */
+#define NUM_SECS_ROLLING_MEASURE 10
+/** How large are the intervals for which we track and report bandwidth use? */
+#define NUM_SECS_BW_SUM_INTERVAL (24*60*60)
+/** How far in the past do we remember and publish bandwidth use? */
+#define NUM_SECS_BW_SUM_IS_VALID (5*24*60*60)
+/** How many bandwidth usage intervals do we remember? (derived) */
+#define NUM_TOTALS (NUM_SECS_BW_SUM_IS_VALID/NUM_SECS_BW_SUM_INTERVAL)
+
+/** Structure to track bandwidth use, and remember the maxima for a given
+ * time period.
+ */
+typedef struct bw_array_t {
+  /** Observation array: Total number of bytes transferred in each of the last
+   * NUM_SECS_ROLLING_MEASURE seconds. This is used as a circular array. */
+  uint64_t obs[NUM_SECS_ROLLING_MEASURE];
+  int cur_obs_idx; /**< Current position in obs. */
+  time_t cur_obs_time; /**< Time represented in obs[cur_obs_idx] */
+  uint64_t total_obs; /**< Total for all members of obs except
+                       * obs[cur_obs_idx] */
+  uint64_t max_total; /**< Largest value that total_obs has taken on in the
+                       * current period. */
+  uint64_t total_in_period; /**< Total bytes transferred in the current
+                             * period. */
+
+  /** When does the next period begin? */
+  time_t next_period;
+  /** Where in 'maxima' should the maximum bandwidth usage for the current
+   * period be stored? */
+  int next_max_idx;
+  /** How many values in maxima/totals have been set ever? */
+  int num_maxes_set;
+  /** Circular array of the maximum
+   * bandwidth-per-NUM_SECS_ROLLING_MEASURE usage for the last
+   * NUM_TOTALS periods */
+  uint64_t maxima[NUM_TOTALS];
+  /** Circular array of the total bandwidth usage for the last NUM_TOTALS
+   * periods */
+  uint64_t totals[NUM_TOTALS];
+} bw_array_t;
+
+/** Shift the current period of b forward by one. */
+static void
+commit_max(bw_array_t *b)
+{
+  /* Store total from current period. */
+  b->totals[b->next_max_idx] = b->total_in_period;
+  /* Store maximum from current period. */
+  b->maxima[b->next_max_idx++] = b->max_total;
+  /* Advance next_period and next_max_idx */
+  b->next_period += NUM_SECS_BW_SUM_INTERVAL;
+  if (b->next_max_idx == NUM_TOTALS)
+    b->next_max_idx = 0;
+  if (b->num_maxes_set < NUM_TOTALS)
+    ++b->num_maxes_set;
+  /* Reset max_total. */
+  b->max_total = 0;
+  /* Reset total_in_period. */
+  b->total_in_period = 0;
+}
+
+/** Shift the current observation time of <b>b</b> forward by one second. */
+static inline void
+advance_obs(bw_array_t *b)
+{
+  int nextidx;
+  uint64_t total;
+
+  /* Calculate the total bandwidth for the last NUM_SECS_ROLLING_MEASURE
+   * seconds; adjust max_total as needed.*/
+  total = b->total_obs + b->obs[b->cur_obs_idx];
+  if (total > b->max_total)
+    b->max_total = total;
+
+  nextidx = b->cur_obs_idx+1;
+  if (nextidx == NUM_SECS_ROLLING_MEASURE)
+    nextidx = 0;
+
+  b->total_obs = total - b->obs[nextidx];
+  b->obs[nextidx]=0;
+  b->cur_obs_idx = nextidx;
+
+  if (++b->cur_obs_time >= b->next_period)
+    commit_max(b);
+}
+
+/** Add <b>n</b> bytes to the number of bytes in <b>b</b> for second
+ * <b>when</b>. */
+static inline void
+add_obs(bw_array_t *b, time_t when, uint64_t n)
+{
+  if (when < b->cur_obs_time)
+    return; /* Don't record data in the past. */
+
+  /* If we're currently adding observations for an earlier second than
+   * 'when', advance b->cur_obs_time and b->cur_obs_idx by an
+   * appropriate number of seconds, and do all the other housekeeping. */
+  while (when > b->cur_obs_time) {
+    /* Doing this one second at a time is potentially inefficient, if we start
+       with a state file that is very old.  Fortunately, it doesn't seem to
+       show up in profiles, so we can just ignore it for now. */
+    advance_obs(b);
+  }
+
+  b->obs[b->cur_obs_idx] += n;
+  b->total_in_period += n;
+}
+
+/** Allocate, initialize, and return a new bw_array. */
+static bw_array_t *
+bw_array_new(void)
+{
+  bw_array_t *b;
+  time_t start;
+  b = tor_malloc_zero(sizeof(bw_array_t));
+  rephist_total_alloc += sizeof(bw_array_t);
+  start = time(NULL);
+  b->cur_obs_time = start;
+  b->next_period = start + NUM_SECS_BW_SUM_INTERVAL;
+  return b;
+}
+
+#define bw_array_free(val) \
+  FREE_AND_NULL(bw_array_t, bw_array_free_, (val))
+
+/** Free storage held by bandwidth array <b>b</b>. */
+static void
+bw_array_free_(bw_array_t *b)
+{
+  if (!b) {
+    return;
+  }
+
+  rephist_total_alloc -= sizeof(bw_array_t);
+  tor_free(b);
+}
+
+/** Recent history of bandwidth observations for read operations. */
+static bw_array_t *read_array = NULL;
+/** Recent history of bandwidth observations for write operations. */
+static bw_array_t *write_array = NULL;
+/** Recent history of bandwidth observations for read operations for the
+    directory protocol. */
+static bw_array_t *dir_read_array = NULL;
+/** Recent history of bandwidth observations for write operations for the
+    directory protocol. */
+static bw_array_t *dir_write_array = NULL;
+
+/** Set up [dir-]read_array and [dir-]write_array, freeing them if they
+ * already exist. */
+static void
+bw_arrays_init(void)
+{
+  bw_array_free(read_array);
+  bw_array_free(write_array);
+  bw_array_free(dir_read_array);
+  bw_array_free(dir_write_array);
+
+  read_array = bw_array_new();
+  write_array = bw_array_new();
+  dir_read_array = bw_array_new();
+  dir_write_array = bw_array_new();
+}
+
+/** Remember that we read <b>num_bytes</b> bytes in second <b>when</b>.
+ *
+ * Add num_bytes to the current running total for <b>when</b>.
+ *
+ * <b>when</b> can go back to time, but it's safe to ignore calls
+ * earlier than the latest <b>when</b> you've heard of.
+ */
+void
+rep_hist_note_bytes_written(size_t num_bytes, time_t when)
+{
+/* Maybe a circular array for recent seconds, and step to a new point
+ * every time a new second shows up. Or simpler is to just to have
+ * a normal array and push down each item every second; it's short.
+ */
+/* When a new second has rolled over, compute the sum of the bytes we've
+ * seen over when-1 to when-1-NUM_SECS_ROLLING_MEASURE, and stick it
+ * somewhere. See rep_hist_bandwidth_assess() below.
+ */
+  add_obs(write_array, when, num_bytes);
+}
+
+/** Remember that we wrote <b>num_bytes</b> bytes in second <b>when</b>.
+ * (like rep_hist_note_bytes_written() above)
+ */
+void
+rep_hist_note_bytes_read(size_t num_bytes, time_t when)
+{
+/* if we're smart, we can make this func and the one above share code */
+  add_obs(read_array, when, num_bytes);
+}
+
+/** Remember that we wrote <b>num_bytes</b> directory bytes in second
+ * <b>when</b>. (like rep_hist_note_bytes_written() above)
+ */
+void
+rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when)
+{
+  add_obs(dir_write_array, when, num_bytes);
+}
+
+/** Remember that we read <b>num_bytes</b> directory bytes in second
+ * <b>when</b>. (like rep_hist_note_bytes_written() above)
+ */
+void
+rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when)
+{
+  add_obs(dir_read_array, when, num_bytes);
+}
+
+/** Helper: Return the largest value in b->maxima.  (This is equal to the
+ * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last
+ * NUM_SECS_BW_SUM_IS_VALID seconds.)
+ */
+static uint64_t
+find_largest_max(bw_array_t *b)
+{
+  int i;
+  uint64_t max;
+  max=0;
+  for (i=0; i<NUM_TOTALS; ++i) {
+    if (b->maxima[i]>max)
+      max = b->maxima[i];
+  }
+  return max;
+}
+
+/** Find the largest sums in the past NUM_SECS_BW_SUM_IS_VALID (roughly)
+ * seconds. Find one sum for reading and one for writing. They don't have
+ * to be at the same time.
+ *
+ * Return the smaller of these sums, divided by NUM_SECS_ROLLING_MEASURE.
+ */
+int
+rep_hist_bandwidth_assess(void)
+{
+  uint64_t w,r;
+  r = find_largest_max(read_array);
+  w = find_largest_max(write_array);
+  if (r>w)
+    return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE);
+  else
+    return (int)(((double)r)/NUM_SECS_ROLLING_MEASURE);
+}
+
+/** Print the bandwidth history of b (either [dir-]read_array or
+ * [dir-]write_array) into the buffer pointed to by buf.  The format is
+ * simply comma separated numbers, from oldest to newest.
+ *
+ * It returns the number of bytes written.
+ */
+static size_t
+rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b)
+{
+  char *cp = buf;
+  int i, n;
+  const or_options_t *options = get_options();
+  uint64_t cutoff;
+
+  if (b->num_maxes_set <= b->next_max_idx) {
+    /* We haven't been through the circular array yet; time starts at i=0.*/
+    i = 0;
+  } else {
+    /* We've been around the array at least once.  The next i to be
+       overwritten is the oldest. */
+    i = b->next_max_idx;
+  }
+
+  if (options->RelayBandwidthRate) {
+    /* We don't want to report that we used more bandwidth than the max we're
+     * willing to relay; otherwise everybody will know how much traffic
+     * we used ourself. */
+    cutoff = options->RelayBandwidthRate * NUM_SECS_BW_SUM_INTERVAL;
+  } else {
+    cutoff = UINT64_MAX;
+  }
+
+  for (n=0; n<b->num_maxes_set; ++n,++i) {
+    uint64_t total;
+    if (i >= NUM_TOTALS)
+      i -= NUM_TOTALS;
+    tor_assert(i < NUM_TOTALS);
+    /* Round the bandwidth used down to the nearest 1k. */
+    total = b->totals[i] & ~0x3ff;
+    if (total > cutoff)
+      total = cutoff;
+
+    if (n==(b->num_maxes_set-1))
+      tor_snprintf(cp, len-(cp-buf), "%"PRIu64, (total));
+    else
+      tor_snprintf(cp, len-(cp-buf), "%"PRIu64",", (total));
+    cp += strlen(cp);
+  }
+  return cp-buf;
+}
+
+/** Allocate and return lines for representing this server's bandwidth
+ * history in its descriptor. We publish these lines in our extra-info
+ * descriptor.
+ */
+char *
+rep_hist_get_bandwidth_lines(void)
+{
+  char *buf, *cp;
+  char t[ISO_TIME_LEN+1];
+  int r;
+  bw_array_t *b = NULL;
+  const char *desc = NULL;
+  size_t len;
+
+  /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */
+/* The n,n,n part above. Largest representation of a uint64_t is 20 chars
+ * long, plus the comma. */
+#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS)
+  len = (67+MAX_HIST_VALUE_LEN)*4;
+  buf = tor_malloc_zero(len);
+  cp = buf;
+  for (r=0;r<4;++r) {
+    char tmp[MAX_HIST_VALUE_LEN];
+    size_t slen;
+    switch (r) {
+      case 0:
+        b = write_array;
+        desc = "write-history";
+        break;
+      case 1:
+        b = read_array;
+        desc = "read-history";
+        break;
+      case 2:
+        b = dir_write_array;
+        desc = "dirreq-write-history";
+        break;
+      case 3:
+        b = dir_read_array;
+        desc = "dirreq-read-history";
+        break;
+    }
+    tor_assert(b);
+    slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b);
+    /* If we don't have anything to write, skip to the next entry. */
+    if (slen == 0)
+      continue;
+    format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL);
+    tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ",
+                 desc, t, NUM_SECS_BW_SUM_INTERVAL);
+    cp += strlen(cp);
+    strlcat(cp, tmp, len-(cp-buf));
+    cp += slen;
+    strlcat(cp, "\n", len-(cp-buf));
+    ++cp;
+  }
+  return buf;
+}
+
+/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum
+ * entries of an or_state_t. Done before writing out a new state file. */
+static void
+rep_hist_update_bwhist_state_section(or_state_t *state,
+                                     const bw_array_t *b,
+                                     smartlist_t **s_values,
+                                     smartlist_t **s_maxima,
+                                     time_t *s_begins,
+                                     int *s_interval)
+{
+  int i,j;
+  uint64_t maxval;
+
+  if (*s_values) {
+    SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val));
+    smartlist_free(*s_values);
+  }
+  if (*s_maxima) {
+    SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val));
+    smartlist_free(*s_maxima);
+  }
+  if (! server_mode(get_options())) {
+    /* Clients don't need to store bandwidth history persistently;
+     * force these values to the defaults. */
+    /* FFFF we should pull the default out of config.c's state table,
+     * so we don't have two defaults. */
+    if (*s_begins != 0 || *s_interval != 900) {
+      time_t now = time(NULL);
+      time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600;
+      or_state_mark_dirty(state, save_at);
+    }
+    *s_begins = 0;
+    *s_interval = 900;
+    *s_values = smartlist_new();
+    *s_maxima = smartlist_new();
+    return;
+  }
+  *s_begins = b->next_period;
+  *s_interval = NUM_SECS_BW_SUM_INTERVAL;
+
+  *s_values = smartlist_new();
+  *s_maxima = smartlist_new();
+  /* Set i to first position in circular array */
+  i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx;
+  for (j=0; j < b->num_maxes_set; ++j,++i) {
+    if (i >= NUM_TOTALS)
+      i = 0;
+    smartlist_add_asprintf(*s_values, "%"PRIu64,
+                           (b->totals[i] & ~0x3ff));
+    maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE;
+    smartlist_add_asprintf(*s_maxima, "%"PRIu64,
+                           (maxval & ~0x3ff));
+  }
+  smartlist_add_asprintf(*s_values, "%"PRIu64,
+                         (b->total_in_period & ~0x3ff));
+  maxval = b->max_total / NUM_SECS_ROLLING_MEASURE;
+  smartlist_add_asprintf(*s_maxima, "%"PRIu64,
+                         (maxval & ~0x3ff));
+}
+
+/** Update <b>state</b> with the newest bandwidth history. Done before
+ * writing out a new state file. */
+void
+rep_hist_update_state(or_state_t *state)
+{
+#define UPDATE(arrname,st) \
+  rep_hist_update_bwhist_state_section(state,\
+                                       (arrname),\
+                                       &state->BWHistory ## st ## Values, \
+                                       &state->BWHistory ## st ## Maxima, \
+                                       &state->BWHistory ## st ## Ends, \
+                                       &state->BWHistory ## st ## Interval)
+
+  UPDATE(write_array, Write);
+  UPDATE(read_array, Read);
+  UPDATE(dir_write_array, DirWrite);
+  UPDATE(dir_read_array, DirRead);
+
+  if (server_mode(get_options())) {
+    or_state_mark_dirty(state, time(NULL)+(2*3600));
+  }
+#undef UPDATE
+}
+
+/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval
+ * entries in an or_state_t. Done while reading the state file. */
+static int
+rep_hist_load_bwhist_state_section(bw_array_t *b,
+                                   const smartlist_t *s_values,
+                                   const smartlist_t *s_maxima,
+                                   const time_t s_begins,
+                                   const int s_interval)
+{
+  time_t now = time(NULL);
+  int retval = 0;
+  time_t start;
+
+  uint64_t v, mv;
+  int i,ok,ok_m = 0;
+  int have_maxima = s_maxima && s_values &&
+    (smartlist_len(s_values) == smartlist_len(s_maxima));
+
+  if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) {
+    start = s_begins - s_interval*(smartlist_len(s_values));
+    if (start > now)
+      return 0;
+    b->cur_obs_time = start;
+    b->next_period = start + NUM_SECS_BW_SUM_INTERVAL;
+    SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) {
+        const char *maxstr = NULL;
+        v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL);
+        if (have_maxima) {
+          maxstr = smartlist_get(s_maxima, cp_sl_idx);
+          mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL);
+          mv *= NUM_SECS_ROLLING_MEASURE;
+        } else {
+          /* No maxima known; guess average rate to be conservative. */
+          mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE;
+        }
+        if (!ok) {
+          retval = -1;
+          log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp);
+        }
+        if (maxstr && !ok_m) {
+          retval = -1;
+          log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'",
+                     maxstr);
+        }
+
+        if (start < now) {
+          time_t cur_start = start;
+          time_t actual_interval_len = s_interval;
+          uint64_t cur_val = 0;
+          /* Calculate the average per second. This is the best we can do
+           * because our state file doesn't have per-second resolution. */
+          if (start + s_interval > now)
+            actual_interval_len = now - start;
+          cur_val = v / actual_interval_len;
+          /* This is potentially inefficient, but since we don't do it very
+           * often it should be ok. */
+          while (cur_start < start + actual_interval_len) {
+            add_obs(b, cur_start, cur_val);
+            ++cur_start;
+          }
+          b->max_total = mv;
+          /* This will result in some fairly choppy history if s_interval
+           * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */
+          start += actual_interval_len;
+        }
+    } SMARTLIST_FOREACH_END(cp);
+  }
+
+  /* Clean up maxima and observed */
+  for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) {
+    b->obs[i] = 0;
+  }
+  b->total_obs = 0;
+
+  return retval;
+}
+
+/** Set bandwidth history from the state file we just loaded. */
+int
+rep_hist_load_state(or_state_t *state, char **err)
+{
+  int all_ok = 1;
+
+  /* Assert they already have been malloced */
+  tor_assert(read_array && write_array);
+  tor_assert(dir_read_array && dir_write_array);
+
+#define LOAD(arrname,st)                                                \
+  if (rep_hist_load_bwhist_state_section(                               \
+                                (arrname),                              \
+                                state->BWHistory ## st ## Values,       \
+                                state->BWHistory ## st ## Maxima,       \
+                                state->BWHistory ## st ## Ends,         \
+                                state->BWHistory ## st ## Interval)<0)  \
+    all_ok = 0
+
+  LOAD(write_array, Write);
+  LOAD(read_array, Read);
+  LOAD(dir_write_array, DirWrite);
+  LOAD(dir_read_array, DirRead);
+
+#undef LOAD
+  if (!all_ok) {
+    *err = tor_strdup("Parsing of bandwidth history values failed");
+    /* and create fresh arrays */
+    bw_arrays_init();
+    return -1;
+  }
+  return 0;
+}
+
+/*********************************************************************/
+
+/** A single predicted port: used to remember which ports we've made
+ * connections to, so that we can try to keep making circuits that can handle
+ * those ports. */
+typedef struct predicted_port_t {
+  /** The port we connected to */
+  uint16_t port;
+  /** The time at which we last used it */
+  time_t time;
+} predicted_port_t;
+
+/** A list of port numbers that have been used recently. */
+static smartlist_t *predicted_ports_list=NULL;
+/** How long do we keep predicting circuits? */
+static int prediction_timeout=0;
+/** When was the last time we added a prediction entry (HS or port) */
+static time_t last_prediction_add_time=0;
+
+/**
+ * How much time left until we stop predicting circuits?
+ */
+int
+predicted_ports_prediction_time_remaining(time_t now)
+{
+  time_t idle_delta;
+
+  /* Protect against overflow of return value. This can happen if the clock
+   * jumps backwards in time. Update the last prediction time (aka last
+   * active time) to prevent it. This update is preferable to using monotonic
+   * time because it prevents clock jumps into the past from simply causing
+   * very long idle timeouts while the monotonic time stands still. */
+  if (last_prediction_add_time > now) {
+    last_prediction_add_time = now;
+    idle_delta = 0;
+  } else {
+    idle_delta = now - last_prediction_add_time;
+  }
+
+  /* Protect against underflow of the return value. This can happen for very
+   * large periods of inactivity/system sleep. */
+  if (idle_delta > prediction_timeout)
+    return 0;
+
+  if (BUG((prediction_timeout - idle_delta) > INT_MAX)) {
+    return INT_MAX;
+  }
+
+  return (int)(prediction_timeout - idle_delta);
+}
+
+/** We just got an application request for a connection with
+ * port <b>port</b>. Remember it for the future, so we can keep
+ * some circuits open that will exit to this port.
+ */
+static void
+add_predicted_port(time_t now, uint16_t port)
+{
+  predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t));
+
+  //  If the list is empty, re-randomize predicted ports lifetime
+  if (!any_predicted_circuits(now)) {
+    prediction_timeout = channelpadding_get_circuits_available_timeout();
+  }
+
+  last_prediction_add_time = now;
+
+  log_info(LD_CIRC,
+          "New port prediction added. Will continue predictive circ building "
+          "for %d more seconds.",
+          predicted_ports_prediction_time_remaining(now));
+
+  pp->port = port;
+  pp->time = now;
+  rephist_total_alloc += sizeof(*pp);
+  smartlist_add(predicted_ports_list, pp);
+}
+
+/**
+ * Allocate whatever memory and structs are needed for predicting
+ * which ports will be used. Also seed it with port 80, so we'll build
+ * circuits on start-up.
+ */
+static void
+predicted_ports_alloc(void)
+{
+  predicted_ports_list = smartlist_new();
+}
+
+void
+predicted_ports_init(void)
+{
+  add_predicted_port(time(NULL), 443); // Add a port to get us started
+}
+
+/** Free whatever memory is needed for predicting which ports will
+ * be used.
+ */
+static void
+predicted_ports_free_all(void)
+{
+  rephist_total_alloc -=
+    smartlist_len(predicted_ports_list)*sizeof(predicted_port_t);
+  SMARTLIST_FOREACH(predicted_ports_list, predicted_port_t *,
+                    pp, tor_free(pp));
+  smartlist_free(predicted_ports_list);
+}
+
+/** Remember that <b>port</b> has been asked for as of time <b>now</b>.
+ * This is used for predicting what sorts of streams we'll make in the
+ * future and making exit circuits to anticipate that.
+ */
+void
+rep_hist_note_used_port(time_t now, uint16_t port)
+{
+  tor_assert(predicted_ports_list);
+
+  if (!port) /* record nothing */
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
+    if (pp->port == port) {
+      pp->time = now;
+
+      last_prediction_add_time = now;
+      log_info(LD_CIRC,
+               "New port prediction added. Will continue predictive circ "
+               "building for %d more seconds.",
+               predicted_ports_prediction_time_remaining(now));
+      return;
+    }
+  } SMARTLIST_FOREACH_END(pp);
+  /* it's not there yet; we need to add it */
+  add_predicted_port(now, port);
+}
+
+/** Return a newly allocated pointer to a list of uint16_t * for ports that
+ * are likely to be asked for in the near future.
+ */
+smartlist_t *
+rep_hist_get_predicted_ports(time_t now)
+{
+  int predicted_circs_relevance_time;
+  smartlist_t *out = smartlist_new();
+  tor_assert(predicted_ports_list);
+
+  predicted_circs_relevance_time = prediction_timeout;
+
+  /* clean out obsolete entries */
+  SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
+    if (pp->time + predicted_circs_relevance_time < now) {
+      log_debug(LD_CIRC, "Expiring predicted port %d", pp->port);
+
+      rephist_total_alloc -= sizeof(predicted_port_t);
+      tor_free(pp);
+      SMARTLIST_DEL_CURRENT(predicted_ports_list, pp);
+    } else {
+      smartlist_add(out, tor_memdup(&pp->port, sizeof(uint16_t)));
+    }
+  } SMARTLIST_FOREACH_END(pp);
+  return out;
+}
+
+/**
+ * Take a list of uint16_t *, and remove every port in the list from the
+ * current list of predicted ports.
+ */
+void
+rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports)
+{
+  /* Let's do this on O(N), not O(N^2). */
+  bitarray_t *remove_ports = bitarray_init_zero(UINT16_MAX);
+  SMARTLIST_FOREACH(rmv_ports, const uint16_t *, p,
+                    bitarray_set(remove_ports, *p));
+  SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
+    if (bitarray_is_set(remove_ports, pp->port)) {
+      tor_free(pp);
+      rephist_total_alloc -= sizeof(*pp);
+      SMARTLIST_DEL_CURRENT(predicted_ports_list, pp);
+    }
+  } SMARTLIST_FOREACH_END(pp);
+  bitarray_free(remove_ports);
+}
+
+/** The user asked us to do a resolve. Rather than keeping track of
+ * timings and such of resolves, we fake it for now by treating
+ * it the same way as a connection to port 80. This way we will continue
+ * to have circuits lying around if the user only uses Tor for resolves.
+ */
+void
+rep_hist_note_used_resolve(time_t now)
+{
+  rep_hist_note_used_port(now, 80);
+}
+
+/** The last time at which we needed an internal circ. */
+static time_t predicted_internal_time = 0;
+/** The last time we needed an internal circ with good uptime. */
+static time_t predicted_internal_uptime_time = 0;
+/** The last time we needed an internal circ with good capacity. */
+static time_t predicted_internal_capacity_time = 0;
+
+/** Remember that we used an internal circ at time <b>now</b>. */
+void
+rep_hist_note_used_internal(time_t now, int need_uptime, int need_capacity)
+{
+  // If the list is empty, re-randomize predicted ports lifetime
+  if (!any_predicted_circuits(now)) {
+    prediction_timeout = channelpadding_get_circuits_available_timeout();
+  }
+
+  last_prediction_add_time = now;
+
+  log_info(LD_CIRC,
+          "New port prediction added. Will continue predictive circ building "
+          "for %d more seconds.",
+          predicted_ports_prediction_time_remaining(now));
+
+  predicted_internal_time = now;
+  if (need_uptime)
+    predicted_internal_uptime_time = now;
+  if (need_capacity)
+    predicted_internal_capacity_time = now;
+}
+
+/** Return 1 if we've used an internal circ recently; else return 0. */
+int
+rep_hist_get_predicted_internal(time_t now, int *need_uptime,
+                                int *need_capacity)
+{
+  int predicted_circs_relevance_time;
+
+  predicted_circs_relevance_time = prediction_timeout;
+
+  if (!predicted_internal_time) { /* initialize it */
+    predicted_internal_time = now;
+    predicted_internal_uptime_time = now;
+    predicted_internal_capacity_time = now;
+  }
+  if (predicted_internal_time + predicted_circs_relevance_time < now)
+    return 0; /* too long ago */
+  if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now)
+    *need_uptime = 1;
+  // Always predict that we need capacity.
+  *need_capacity = 1;
+  return 1;
+}
+
+/** Any ports used lately? These are pre-seeded if we just started
+ * up or if we're running a hidden service. */
+int
+any_predicted_circuits(time_t now)
+{
+  int predicted_circs_relevance_time;
+  predicted_circs_relevance_time = prediction_timeout;
+
+  return smartlist_len(predicted_ports_list) ||
+         predicted_internal_time + predicted_circs_relevance_time >= now;
+}
+
+/** Return 1 if we have no need for circuits currently, else return 0. */
+int
+rep_hist_circbuilding_dormant(time_t now)
+{
+  const or_options_t *options = get_options();
+
+  if (any_predicted_circuits(now))
+    return 0;
+
+  /* see if we'll still need to build testing circuits */
+  if (server_mode(options) &&
+      (!check_whether_orport_reachable(options) ||
+       !circuit_enough_testing_circs()))
+    return 0;
+  if (!check_whether_dirport_reachable(options))
+    return 0;
+
+  return 1;
+}
+
+/*** Exit port statistics ***/
+
+/* Some constants */
+/** To what multiple should byte numbers be rounded up? */
+#define EXIT_STATS_ROUND_UP_BYTES 1024
+/** To what multiple should stream counts be rounded up? */
+#define EXIT_STATS_ROUND_UP_STREAMS 4
+/** Number of TCP ports */
+#define EXIT_STATS_NUM_PORTS 65536
+/** Top n ports that will be included in exit stats. */
+#define EXIT_STATS_TOP_N_PORTS 10
+
+/* The following data structures are arrays and no fancy smartlists or maps,
+ * so that all write operations can be done in constant time. This comes at
+ * the price of some memory (1.25 MB) and linear complexity when writing
+ * stats for measuring relays. */
+/** Number of bytes read in current period by exit port */
+static uint64_t *exit_bytes_read = NULL;
+/** Number of bytes written in current period by exit port */
+static uint64_t *exit_bytes_written = NULL;
+/** Number of streams opened in current period by exit port */
+static uint32_t *exit_streams = NULL;
+
+/** Start time of exit stats or 0 if we're not collecting exit stats. */
+static time_t start_of_exit_stats_interval;
+
+/** Initialize exit port stats. */
+void
+rep_hist_exit_stats_init(time_t now)
+{
+  start_of_exit_stats_interval = now;
+  exit_bytes_read = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
+  exit_bytes_written = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
+  exit_streams = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint32_t));
+}
+
+/** Reset counters for exit port statistics. */
+void
+rep_hist_reset_exit_stats(time_t now)
+{
+  start_of_exit_stats_interval = now;
+  memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
+  memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
+  memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t));
+}
+
+/** Stop collecting exit port stats in a way that we can re-start doing
+ * so in rep_hist_exit_stats_init(). */
+void
+rep_hist_exit_stats_term(void)
+{
+  start_of_exit_stats_interval = 0;
+  tor_free(exit_bytes_read);
+  tor_free(exit_bytes_written);
+  tor_free(exit_streams);
+}
+
+/** Helper for qsort: compare two ints.  Does not handle overflow properly,
+ * but works fine for sorting an array of port numbers, which is what we use
+ * it for. */
+static int
+compare_int_(const void *x, const void *y)
+{
+  return (*(int*)x - *(int*)y);
+}
+
+/** Return a newly allocated string containing the exit port statistics
+ * until <b>now</b>, or NULL if we're not collecting exit stats. Caller
+ * must ensure start_of_exit_stats_interval is in the past. */
+char *
+rep_hist_format_exit_stats(time_t now)
+{
+  int i, j, top_elements = 0, cur_min_idx = 0, cur_port;
+  uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS];
+  int top_ports[EXIT_STATS_TOP_N_PORTS];
+  uint64_t cur_bytes = 0, other_read = 0, other_written = 0,
+           total_read = 0, total_written = 0;
+  uint32_t total_streams = 0, other_streams = 0;
+  smartlist_t *written_strings, *read_strings, *streams_strings;
+  char *written_string, *read_string, *streams_string;
+  char t[ISO_TIME_LEN+1];
+  char *result;
+
+  if (!start_of_exit_stats_interval)
+    return NULL; /* Not initialized. */
+
+  tor_assert(now >= start_of_exit_stats_interval);
+
+  /* Go through all ports to find the n ports that saw most written and
+   * read bytes.
+   *
+   * Invariant: at the end of the loop for iteration i,
+   *    total_read is the sum of all exit_bytes_read[0..i]
+   *    total_written is the sum of all exit_bytes_written[0..i]
+   *    total_stream is the sum of all exit_streams[0..i]
+   *
+   *    top_elements = MAX(EXIT_STATS_TOP_N_PORTS,
+   *                  #{j | 0 <= j <= i && volume(i) > 0})
+   *
+   *    For all 0 <= j < top_elements,
+   *        top_bytes[j] > 0
+   *        0 <= top_ports[j] <= 65535
+   *        top_bytes[j] = volume(top_ports[j])
+   *
+   *    There is no j in 0..i and k in 0..top_elements such that:
+   *        volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements]
+   *
+   *    There is no j!=cur_min_idx in 0..top_elements such that:
+   *        top_bytes[j] < top_bytes[cur_min_idx]
+   *
+   * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x]
+   *
+   * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS)
+   */
+  for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
+    total_read += exit_bytes_read[i];
+    total_written += exit_bytes_written[i];
+    total_streams += exit_streams[i];
+    cur_bytes = exit_bytes_read[i] + exit_bytes_written[i];
+    if (cur_bytes == 0) {
+      continue;
+    }
+    if (top_elements < EXIT_STATS_TOP_N_PORTS) {
+      top_bytes[top_elements] = cur_bytes;
+      top_ports[top_elements++] = i;
+    } else if (cur_bytes > top_bytes[cur_min_idx]) {
+      top_bytes[cur_min_idx] = cur_bytes;
+      top_ports[cur_min_idx] = i;
+    } else {
+      continue;
+    }
+    cur_min_idx = 0;
+    for (j = 1; j < top_elements; j++) {
+      if (top_bytes[j] < top_bytes[cur_min_idx]) {
+        cur_min_idx = j;
+      }
+    }
+  }
+
+  /* Add observations of top ports to smartlists. */
+  written_strings = smartlist_new();
+  read_strings = smartlist_new();
+  streams_strings = smartlist_new();
+  other_read = total_read;
+  other_written = total_written;
+  other_streams = total_streams;
+  /* Sort the ports; this puts them out of sync with top_bytes, but we
+   * won't be using top_bytes again anyway */
+  qsort(top_ports, top_elements, sizeof(int), compare_int_);
+  for (j = 0; j < top_elements; j++) {
+    cur_port = top_ports[j];
+    if (exit_bytes_written[cur_port] > 0) {
+      uint64_t num = round_uint64_to_next_multiple_of(
+                     exit_bytes_written[cur_port],
+                     EXIT_STATS_ROUND_UP_BYTES);
+      num /= 1024;
+      smartlist_add_asprintf(written_strings, "%d=%"PRIu64,
+                             cur_port, (num));
+      other_written -= exit_bytes_written[cur_port];
+    }
+    if (exit_bytes_read[cur_port] > 0) {
+      uint64_t num = round_uint64_to_next_multiple_of(
+                     exit_bytes_read[cur_port],
+                     EXIT_STATS_ROUND_UP_BYTES);
+      num /= 1024;
+      smartlist_add_asprintf(read_strings, "%d=%"PRIu64,
+                             cur_port, (num));
+      other_read -= exit_bytes_read[cur_port];
+    }
+    if (exit_streams[cur_port] > 0) {
+      uint32_t num = round_uint32_to_next_multiple_of(
+                     exit_streams[cur_port],
+                     EXIT_STATS_ROUND_UP_STREAMS);
+      smartlist_add_asprintf(streams_strings, "%d=%u", cur_port, num);
+      other_streams -= exit_streams[cur_port];
+    }
+  }
+
+  /* Add observations of other ports in a single element. */
+  other_written = round_uint64_to_next_multiple_of(other_written,
+                  EXIT_STATS_ROUND_UP_BYTES);
+  other_written /= 1024;
+  smartlist_add_asprintf(written_strings, "other=%"PRIu64,
+                         (other_written));
+  other_read = round_uint64_to_next_multiple_of(other_read,
+               EXIT_STATS_ROUND_UP_BYTES);
+  other_read /= 1024;
+  smartlist_add_asprintf(read_strings, "other=%"PRIu64,
+                         (other_read));
+  other_streams = round_uint32_to_next_multiple_of(other_streams,
+                  EXIT_STATS_ROUND_UP_STREAMS);
+  smartlist_add_asprintf(streams_strings, "other=%u", other_streams);
+
+  /* Join all observations in single strings. */
+  written_string = smartlist_join_strings(written_strings, ",", 0, NULL);
+  read_string = smartlist_join_strings(read_strings, ",", 0, NULL);
+  streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL);
+  SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp));
+  SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp));
+  SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp));
+  smartlist_free(written_strings);
+  smartlist_free(read_strings);
+  smartlist_free(streams_strings);
+
+  /* Put everything together. */
+  format_iso_time(t, now);
+  tor_asprintf(&result, "exit-stats-end %s (%d s)\n"
+               "exit-kibibytes-written %s\n"
+               "exit-kibibytes-read %s\n"
+               "exit-streams-opened %s\n",
+               t, (unsigned) (now - start_of_exit_stats_interval),
+               written_string,
+               read_string,
+               streams_string);
+  tor_free(written_string);
+  tor_free(read_string);
+  tor_free(streams_string);
+  return result;
+}
+
+/** If 24 hours have passed since the beginning of the current exit port
+ * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly
+ * overwriting an existing file) and reset counters.  Return when we would
+ * next want to write exit stats or 0 if we never want to write. */
+time_t
+rep_hist_exit_stats_write(time_t now)
+{
+  char *str = NULL;
+
+  if (!start_of_exit_stats_interval)
+    return 0; /* Not initialized. */
+  if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now)
+    goto done; /* Not ready to write. */
+
+  log_info(LD_HIST, "Writing exit port statistics to disk.");
+
+  /* Generate history string. */
+  str = rep_hist_format_exit_stats(now);
+
+  /* Reset counters. */
+  rep_hist_reset_exit_stats(now);
+
+  /* Try to write to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "exit-stats", str, "exit port statistics");
+  }
+
+ done:
+  tor_free(str);
+  return start_of_exit_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b>
+ * bytes to/from an exit connection to <b>port</b>. */
+void
+rep_hist_note_exit_bytes(uint16_t port, size_t num_written,
+                         size_t num_read)
+{
+  if (!start_of_exit_stats_interval)
+    return; /* Not initialized. */
+  exit_bytes_written[port] += num_written;
+  exit_bytes_read[port] += num_read;
+  log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an "
+            "exit connection to port %d.",
+            (unsigned long)num_written, (unsigned long)num_read, port);
+}
+
+/** Note that we opened an exit stream to <b>port</b>. */
+void
+rep_hist_note_exit_stream_opened(uint16_t port)
+{
+  if (!start_of_exit_stats_interval)
+    return; /* Not initialized. */
+  exit_streams[port]++;
+  log_debug(LD_HIST, "Opened exit stream to port %d", port);
+}
+
+/*** cell statistics ***/
+
+/** Start of the current buffer stats interval or 0 if we're not
+ * collecting buffer statistics. */
+static time_t start_of_buffer_stats_interval;
+
+/** Initialize buffer stats. */
+void
+rep_hist_buffer_stats_init(time_t now)
+{
+  start_of_buffer_stats_interval = now;
+}
+
+/** Statistics from a single circuit.  Collected when the circuit closes, or
+ * when we flush statistics to disk. */
+typedef struct circ_buffer_stats_t {
+  /** Average number of cells in the circuit's queue */
+  double mean_num_cells_in_queue;
+  /** Average time a cell waits in the queue. */
+  double mean_time_cells_in_queue;
+  /** Total number of cells sent over this circuit */
+  uint32_t processed_cells;
+} circ_buffer_stats_t;
+
+/** List of circ_buffer_stats_t. */
+static smartlist_t *circuits_for_buffer_stats = NULL;
+
+/** Remember cell statistics <b>mean_num_cells_in_queue</b>,
+ * <b>mean_time_cells_in_queue</b>, and <b>processed_cells</b> of a
+ * circuit. */
+void
+rep_hist_add_buffer_stats(double mean_num_cells_in_queue,
+    double mean_time_cells_in_queue, uint32_t processed_cells)
+{
+  circ_buffer_stats_t *stats;
+  if (!start_of_buffer_stats_interval)
+    return; /* Not initialized. */
+  stats = tor_malloc_zero(sizeof(circ_buffer_stats_t));
+  stats->mean_num_cells_in_queue = mean_num_cells_in_queue;
+  stats->mean_time_cells_in_queue = mean_time_cells_in_queue;
+  stats->processed_cells = processed_cells;
+  if (!circuits_for_buffer_stats)
+    circuits_for_buffer_stats = smartlist_new();
+  smartlist_add(circuits_for_buffer_stats, stats);
+}
+
+/** Remember cell statistics for circuit <b>circ</b> at time
+ * <b>end_of_interval</b> and reset cell counters in case the circuit
+ * remains open in the next measurement interval. */
+void
+rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
+{
+  time_t start_of_interval;
+  int interval_length;
+  or_circuit_t *orcirc;
+  double mean_num_cells_in_queue, mean_time_cells_in_queue;
+  uint32_t processed_cells;
+  if (CIRCUIT_IS_ORIGIN(circ))
+    return;
+  orcirc = TO_OR_CIRCUIT(circ);
+  if (!orcirc->processed_cells)
+    return;
+  start_of_interval = (circ->timestamp_created.tv_sec >
+                       start_of_buffer_stats_interval) ?
+        (time_t)circ->timestamp_created.tv_sec :
+        start_of_buffer_stats_interval;
+  interval_length = (int) (end_of_interval - start_of_interval);
+  if (interval_length <= 0)
+    return;
+  processed_cells = orcirc->processed_cells;
+  /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */
+  mean_num_cells_in_queue = (double) orcirc->total_cell_waiting_time /
+      (double) interval_length / 1000.0 / 2.0;
+  mean_time_cells_in_queue =
+      (double) orcirc->total_cell_waiting_time /
+      (double) orcirc->processed_cells;
+  orcirc->total_cell_waiting_time = 0;
+  orcirc->processed_cells = 0;
+  rep_hist_add_buffer_stats(mean_num_cells_in_queue,
+                            mean_time_cells_in_queue,
+                            processed_cells);
+}
+
+/** Sorting helper: return -1, 1, or 0 based on comparison of two
+ * circ_buffer_stats_t */
+static int
+buffer_stats_compare_entries_(const void **_a, const void **_b)
+{
+  const circ_buffer_stats_t *a = *_a, *b = *_b;
+  if (a->processed_cells < b->processed_cells)
+    return 1;
+  else if (a->processed_cells > b->processed_cells)
+    return -1;
+  else
+    return 0;
+}
+
+/** Stop collecting cell stats in a way that we can re-start doing so in
+ * rep_hist_buffer_stats_init(). */
+void
+rep_hist_buffer_stats_term(void)
+{
+  rep_hist_reset_buffer_stats(0);
+}
+
+/** Clear history of circuit statistics and set the measurement interval
+ * start to <b>now</b>. */
+void
+rep_hist_reset_buffer_stats(time_t now)
+{
+  if (!circuits_for_buffer_stats)
+    circuits_for_buffer_stats = smartlist_new();
+  SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *,
+      stats, tor_free(stats));
+  smartlist_clear(circuits_for_buffer_stats);
+  start_of_buffer_stats_interval = now;
+}
+
+/** Return a newly allocated string containing the buffer statistics until
+ * <b>now</b>, or NULL if we're not collecting buffer stats. Caller must
+ * ensure start_of_buffer_stats_interval is in the past. */
+char *
+rep_hist_format_buffer_stats(time_t now)
+{
+#define SHARES 10
+  uint64_t processed_cells[SHARES];
+  uint32_t circs_in_share[SHARES];
+  int number_of_circuits, i;
+  double queued_cells[SHARES], time_in_queue[SHARES];
+  smartlist_t *processed_cells_strings, *queued_cells_strings,
+              *time_in_queue_strings;
+  char *processed_cells_string, *queued_cells_string,
+       *time_in_queue_string;
+  char t[ISO_TIME_LEN+1];
+  char *result;
+
+  if (!start_of_buffer_stats_interval)
+    return NULL; /* Not initialized. */
+
+  tor_assert(now >= start_of_buffer_stats_interval);
+
+  /* Calculate deciles if we saw at least one circuit. */
+  memset(processed_cells, 0, SHARES * sizeof(uint64_t));
+  memset(circs_in_share, 0, SHARES * sizeof(uint32_t));
+  memset(queued_cells, 0, SHARES * sizeof(double));
+  memset(time_in_queue, 0, SHARES * sizeof(double));
+  if (!circuits_for_buffer_stats)
+    circuits_for_buffer_stats = smartlist_new();
+  number_of_circuits = smartlist_len(circuits_for_buffer_stats);
+  if (number_of_circuits > 0) {
+    smartlist_sort(circuits_for_buffer_stats,
+                   buffer_stats_compare_entries_);
+    i = 0;
+    SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats,
+                            circ_buffer_stats_t *, stats)
+    {
+      int share = i++ * SHARES / number_of_circuits;
+      processed_cells[share] += stats->processed_cells;
+      queued_cells[share] += stats->mean_num_cells_in_queue;
+      time_in_queue[share] += stats->mean_time_cells_in_queue;
+      circs_in_share[share]++;
+    }
+    SMARTLIST_FOREACH_END(stats);
+  }
+
+  /* Write deciles to strings. */
+  processed_cells_strings = smartlist_new();
+  queued_cells_strings = smartlist_new();
+  time_in_queue_strings = smartlist_new();
+  for (i = 0; i < SHARES; i++) {
+    smartlist_add_asprintf(processed_cells_strings,
+                           "%"PRIu64, !circs_in_share[i] ? 0 :
+                           (processed_cells[i] /
+                           circs_in_share[i]));
+  }
+  for (i = 0; i < SHARES; i++) {
+    smartlist_add_asprintf(queued_cells_strings, "%.2f",
+                           circs_in_share[i] == 0 ? 0.0 :
+                             queued_cells[i] / (double) circs_in_share[i]);
+  }
+  for (i = 0; i < SHARES; i++) {
+    smartlist_add_asprintf(time_in_queue_strings, "%.0f",
+                           circs_in_share[i] == 0 ? 0.0 :
+                             time_in_queue[i] / (double) circs_in_share[i]);
+  }
+
+  /* Join all observations in single strings. */
+  processed_cells_string = smartlist_join_strings(processed_cells_strings,
+                                                  ",", 0, NULL);
+  queued_cells_string = smartlist_join_strings(queued_cells_strings,
+                                               ",", 0, NULL);
+  time_in_queue_string = smartlist_join_strings(time_in_queue_strings,
+                                                ",", 0, NULL);
+  SMARTLIST_FOREACH(processed_cells_strings, char *, cp, tor_free(cp));
+  SMARTLIST_FOREACH(queued_cells_strings, char *, cp, tor_free(cp));
+  SMARTLIST_FOREACH(time_in_queue_strings, char *, cp, tor_free(cp));
+  smartlist_free(processed_cells_strings);
+  smartlist_free(queued_cells_strings);
+  smartlist_free(time_in_queue_strings);
+
+  /* Put everything together. */
+  format_iso_time(t, now);
+  tor_asprintf(&result, "cell-stats-end %s (%d s)\n"
+               "cell-processed-cells %s\n"
+               "cell-queued-cells %s\n"
+               "cell-time-in-queue %s\n"
+               "cell-circuits-per-decile %d\n",
+               t, (unsigned) (now - start_of_buffer_stats_interval),
+               processed_cells_string,
+               queued_cells_string,
+               time_in_queue_string,
+               CEIL_DIV(number_of_circuits, SHARES));
+  tor_free(processed_cells_string);
+  tor_free(queued_cells_string);
+  tor_free(time_in_queue_string);
+  return result;
+#undef SHARES
+}
+
+/** If 24 hours have passed since the beginning of the current buffer
+ * stats period, write buffer stats to $DATADIR/stats/buffer-stats
+ * (possibly overwriting an existing file) and reset counters.  Return
+ * when we would next want to write buffer stats or 0 if we never want to
+ * write. */
+time_t
+rep_hist_buffer_stats_write(time_t now)
+{
+  char *str = NULL;
+
+  if (!start_of_buffer_stats_interval)
+    return 0; /* Not initialized. */
+  if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now)
+    goto done; /* Not ready to write */
+
+  /* Add open circuits to the history. */
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    rep_hist_buffer_stats_add_circ(circ, now);
+  }
+  SMARTLIST_FOREACH_END(circ);
+
+  /* Generate history string. */
+  str = rep_hist_format_buffer_stats(now);
+
+  /* Reset both buffer history and counters of open circuits. */
+  rep_hist_reset_buffer_stats(now);
+
+  /* Try to write to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics");
+  }
+
+ done:
+  tor_free(str);
+  return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/*** Descriptor serving statistics ***/
+
+/** Digestmap to track which descriptors were downloaded this stats
+ *  collection interval. It maps descriptor digest to pointers to 1,
+ *  effectively turning this into a list. */
+static digestmap_t *served_descs = NULL;
+
+/** Number of how many descriptors were downloaded in total during this
+ * interval. */
+static unsigned long total_descriptor_downloads;
+
+/** Start time of served descs stats or 0 if we're not collecting those. */
+static time_t start_of_served_descs_stats_interval;
+
+/** Initialize descriptor stats. */
+void
+rep_hist_desc_stats_init(time_t now)
+{
+  if (served_descs) {
+    log_warn(LD_BUG, "Called rep_hist_desc_stats_init() when desc stats were "
+             "already initialized. This is probably harmless.");
+    return; // Already initialized
+  }
+  served_descs = digestmap_new();
+  total_descriptor_downloads = 0;
+  start_of_served_descs_stats_interval = now;
+}
+
+/** Reset served descs stats to empty, starting a new interval <b>now</b>. */
+static void
+rep_hist_reset_desc_stats(time_t now)
+{
+  rep_hist_desc_stats_term();
+  rep_hist_desc_stats_init(now);
+}
+
+/** Stop collecting served descs stats, so that rep_hist_desc_stats_init() is
+ * safe to be called again. */
+void
+rep_hist_desc_stats_term(void)
+{
+  digestmap_free(served_descs, NULL);
+  served_descs = NULL;
+  start_of_served_descs_stats_interval = 0;
+  total_descriptor_downloads = 0;
+}
+
+/** Helper for rep_hist_desc_stats_write(). Return a newly allocated string
+ * containing the served desc statistics until now, or NULL if we're not
+ * collecting served desc stats. Caller must ensure that now is not before
+ * start_of_served_descs_stats_interval. */
+static char *
+rep_hist_format_desc_stats(time_t now)
+{
+  char t[ISO_TIME_LEN+1];
+  char *result;
+
+  digestmap_iter_t *iter;
+  const char *key;
+  void *val;
+  unsigned size;
+  int *vals, max = 0, q3 = 0, md = 0, q1 = 0, min = 0;
+  int n = 0;
+
+  if (!start_of_served_descs_stats_interval)
+    return NULL;
+
+  size = digestmap_size(served_descs);
+  if (size > 0) {
+    vals = tor_calloc(size, sizeof(int));
+    for (iter = digestmap_iter_init(served_descs);
+         !digestmap_iter_done(iter);
+         iter = digestmap_iter_next(served_descs, iter)) {
+      uintptr_t count;
+      digestmap_iter_get(iter, &key, &val);
+      count = (uintptr_t)val;
+      vals[n++] = (int)count;
+      (void)key;
+    }
+    max = find_nth_int(vals, size, size-1);
+    q3 = find_nth_int(vals, size, (3*size-1)/4);
+    md = find_nth_int(vals, size, (size-1)/2);
+    q1 = find_nth_int(vals, size, (size-1)/4);
+    min = find_nth_int(vals, size, 0);
+    tor_free(vals);
+  }
+
+  format_iso_time(t, now);
+
+  tor_asprintf(&result,
+               "served-descs-stats-end %s (%d s) total=%lu unique=%u "
+               "max=%d q3=%d md=%d q1=%d min=%d\n",
+               t,
+               (unsigned) (now - start_of_served_descs_stats_interval),
+               total_descriptor_downloads,
+               size, max, q3, md, q1, min);
+
+  return result;
+}
+
+/** If WRITE_STATS_INTERVAL seconds have passed since the beginning of
+ * the current served desc stats interval, write the stats to
+ * $DATADIR/stats/served-desc-stats (possibly appending to an existing file)
+ * and reset the state for the next interval. Return when we would next want
+ * to write served desc stats or 0 if we won't want to write. */
+time_t
+rep_hist_desc_stats_write(time_t now)
+{
+  char *filename = NULL, *str = NULL;
+
+  if (!start_of_served_descs_stats_interval)
+    return 0; /* We're not collecting stats. */
+  if (start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL > now)
+    return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
+
+  str = rep_hist_format_desc_stats(now);
+  tor_assert(str != NULL);
+
+  if (check_or_create_data_subdir("stats") < 0) {
+    goto done;
+  }
+  filename = get_datadir_fname2("stats", "served-desc-stats");
+  if (append_bytes_to_file(filename, str, strlen(str), 0) < 0)
+    log_warn(LD_HIST, "Unable to write served descs statistics to disk!");
+
+  rep_hist_reset_desc_stats(now);
+
+ done:
+  tor_free(filename);
+  tor_free(str);
+  return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Called to note that we've served a given descriptor (by
+ * digest). Increments the count of descriptors served, and the number
+ * of times we've served this descriptor. */
+void
+rep_hist_note_desc_served(const char * desc)
+{
+  void *val;
+  uintptr_t count;
+  if (!served_descs)
+    return; // We're not collecting stats
+  val = digestmap_get(served_descs, desc);
+  count = (uintptr_t)val;
+  if (count != INT_MAX)
+    ++count;
+  digestmap_set(served_descs, desc, (void*)count);
+  total_descriptor_downloads++;
+}
+
+/*** Connection statistics ***/
+
+/** Start of the current connection stats interval or 0 if we're not
+ * collecting connection statistics. */
+static time_t start_of_conn_stats_interval;
+
+/** Initialize connection stats. */
+void
+rep_hist_conn_stats_init(time_t now)
+{
+  start_of_conn_stats_interval = now;
+}
+
+/* Count connections that we read and wrote less than these many bytes
+ * from/to as below threshold. */
+#define BIDI_THRESHOLD 20480
+
+/* Count connections that we read or wrote at least this factor as many
+ * bytes from/to than we wrote or read to/from as mostly reading or
+ * writing. */
+#define BIDI_FACTOR 10
+
+/* Interval length in seconds for considering read and written bytes for
+ * connection stats. */
+#define BIDI_INTERVAL 10
+
+/** Start of next BIDI_INTERVAL second interval. */
+static time_t bidi_next_interval = 0;
+
+/** Number of connections that we read and wrote less than BIDI_THRESHOLD
+ * bytes from/to in BIDI_INTERVAL seconds. */
+static uint32_t below_threshold = 0;
+
+/** Number of connections that we read at least BIDI_FACTOR times more
+ * bytes from than we wrote to in BIDI_INTERVAL seconds. */
+static uint32_t mostly_read = 0;
+
+/** Number of connections that we wrote at least BIDI_FACTOR times more
+ * bytes to than we read from in BIDI_INTERVAL seconds. */
+static uint32_t mostly_written = 0;
+
+/** Number of connections that we read and wrote at least BIDI_THRESHOLD
+ * bytes from/to, but not BIDI_FACTOR times more in either direction in
+ * BIDI_INTERVAL seconds. */
+static uint32_t both_read_and_written = 0;
+
+/** Entry in a map from connection ID to the number of read and written
+ * bytes on this connection in a BIDI_INTERVAL second interval. */
+typedef struct bidi_map_entry_t {
+  HT_ENTRY(bidi_map_entry_t) node;
+  uint64_t conn_id; /**< Connection ID */
+  size_t read; /**< Number of read bytes */
+  size_t written; /**< Number of written bytes */
+} bidi_map_entry_t;
+
+/** Map of OR connections together with the number of read and written
+ * bytes in the current BIDI_INTERVAL second interval. */
+static HT_HEAD(bidimap, bidi_map_entry_t) bidi_map =
+     HT_INITIALIZER();
+
+static int
+bidi_map_ent_eq(const bidi_map_entry_t *a, const bidi_map_entry_t *b)
+{
+  return a->conn_id == b->conn_id;
+}
+
+/* DOCDOC bidi_map_ent_hash */
+static unsigned
+bidi_map_ent_hash(const bidi_map_entry_t *entry)
+{
+  return (unsigned) entry->conn_id;
+}
+
+HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
+             bidi_map_ent_eq)
+HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
+             bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/* DOCDOC bidi_map_free */
+static void
+bidi_map_free_all(void)
+{
+  bidi_map_entry_t **ptr, **next, *ent;
+  for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
+    ent = *ptr;
+    next = HT_NEXT_RMV(bidimap, &bidi_map, ptr);
+    tor_free(ent);
+  }
+  HT_CLEAR(bidimap, &bidi_map);
+}
+
+/** Reset counters for conn statistics. */
+void
+rep_hist_reset_conn_stats(time_t now)
+{
+  start_of_conn_stats_interval = now;
+  below_threshold = 0;
+  mostly_read = 0;
+  mostly_written = 0;
+  both_read_and_written = 0;
+  bidi_map_free_all();
+}
+
+/** Stop collecting connection stats in a way that we can re-start doing
+ * so in rep_hist_conn_stats_init(). */
+void
+rep_hist_conn_stats_term(void)
+{
+  rep_hist_reset_conn_stats(0);
+}
+
+/** We read <b>num_read</b> bytes and wrote <b>num_written</b> from/to OR
+ * connection <b>conn_id</b> in second <b>when</b>. If this is the first
+ * observation in a new interval, sum up the last observations. Add bytes
+ * for this connection. */
+void
+rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read,
+                            size_t num_written, time_t when)
+{
+  if (!start_of_conn_stats_interval)
+    return;
+  /* Initialize */
+  if (bidi_next_interval == 0)
+    bidi_next_interval = when + BIDI_INTERVAL;
+  /* Sum up last period's statistics */
+  if (when >= bidi_next_interval) {
+    bidi_map_entry_t **ptr, **next, *ent;
+    for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
+      ent = *ptr;
+      if (ent->read + ent->written < BIDI_THRESHOLD)
+        below_threshold++;
+      else if (ent->read >= ent->written * BIDI_FACTOR)
+        mostly_read++;
+      else if (ent->written >= ent->read * BIDI_FACTOR)
+        mostly_written++;
+      else
+        both_read_and_written++;
+      next = HT_NEXT_RMV(bidimap, &bidi_map, ptr);
+      tor_free(ent);
+    }
+    while (when >= bidi_next_interval)
+      bidi_next_interval += BIDI_INTERVAL;
+    log_info(LD_GENERAL, "%d below threshold, %d mostly read, "
+             "%d mostly written, %d both read and written.",
+             below_threshold, mostly_read, mostly_written,
+             both_read_and_written);
+  }
+  /* Add this connection's bytes. */
+  if (num_read > 0 || num_written > 0) {
+    bidi_map_entry_t *entry, lookup;
+    lookup.conn_id = conn_id;
+    entry = HT_FIND(bidimap, &bidi_map, &lookup);
+    if (entry) {
+      entry->written += num_written;
+      entry->read += num_read;
+    } else {
+      entry = tor_malloc_zero(sizeof(bidi_map_entry_t));
+      entry->conn_id = conn_id;
+      entry->written = num_written;
+      entry->read = num_read;
+      HT_INSERT(bidimap, &bidi_map, entry);
+    }
+  }
+}
+
+/** Return a newly allocated string containing the connection statistics
+ * until <b>now</b>, or NULL if we're not collecting conn stats. Caller must
+ * ensure start_of_conn_stats_interval is in the past. */
+char *
+rep_hist_format_conn_stats(time_t now)
+{
+  char *result, written[ISO_TIME_LEN+1];
+
+  if (!start_of_conn_stats_interval)
+    return NULL; /* Not initialized. */
+
+  tor_assert(now >= start_of_conn_stats_interval);
+
+  format_iso_time(written, now);
+  tor_asprintf(&result, "conn-bi-direct %s (%d s) %d,%d,%d,%d\n",
+               written,
+               (unsigned) (now - start_of_conn_stats_interval),
+               below_threshold,
+               mostly_read,
+               mostly_written,
+               both_read_and_written);
+  return result;
+}
+
+/** If 24 hours have passed since the beginning of the current conn stats
+ * period, write conn stats to $DATADIR/stats/conn-stats (possibly
+ * overwriting an existing file) and reset counters.  Return when we would
+ * next want to write conn stats or 0 if we never want to write. */
+time_t
+rep_hist_conn_stats_write(time_t now)
+{
+  char *str = NULL;
+
+  if (!start_of_conn_stats_interval)
+    return 0; /* Not initialized. */
+  if (start_of_conn_stats_interval + WRITE_STATS_INTERVAL > now)
+    goto done; /* Not ready to write */
+
+  /* Generate history string. */
+  str = rep_hist_format_conn_stats(now);
+
+  /* Reset counters. */
+  rep_hist_reset_conn_stats(now);
+
+  /* Try to write to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "conn-stats", str, "connection statistics");
+  }
+
+ done:
+  tor_free(str);
+  return start_of_conn_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Internal statistics to track how many requests of each type of
+ * handshake we've received, and how many we've assigned to cpuworkers.
+ * Useful for seeing trends in cpu load.
+ * @{ */
+STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
+STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
+/**@}*/
+
+/** A new onionskin (using the <b>type</b> handshake) has arrived. */
+void
+rep_hist_note_circuit_handshake_requested(uint16_t type)
+{
+  if (type <= MAX_ONION_HANDSHAKE_TYPE)
+    onion_handshakes_requested[type]++;
+}
+
+/** We've sent an onionskin (using the <b>type</b> handshake) to a
+ * cpuworker. */
+void
+rep_hist_note_circuit_handshake_assigned(uint16_t type)
+{
+  if (type <= MAX_ONION_HANDSHAKE_TYPE)
+    onion_handshakes_assigned[type]++;
+}
+
+/** Log our onionskin statistics since the last time we were called. */
+void
+rep_hist_log_circuit_handshake_stats(time_t now)
+{
+  (void)now;
+  log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: "
+             "%d/%d TAP, %d/%d NTor.",
+             onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP],
+             onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP],
+             onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR],
+             onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]);
+  memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned));
+  memset(onion_handshakes_requested, 0, sizeof(onion_handshakes_requested));
+}
+
+/* Hidden service statistics section */
+
+/** Start of the current hidden service stats interval or 0 if we're
+ * not collecting hidden service statistics. */
+static time_t start_of_hs_stats_interval;
+
+/** Carries the various hidden service statistics, and any other
+ *  information needed. */
+typedef struct hs_stats_t {
+  /** How many relay cells have we seen as rendezvous points? */
+  uint64_t rp_relay_cells_seen;
+
+  /** Set of unique public key digests we've seen this stat period
+   * (could also be implemented as sorted smartlist). */
+  digestmap_t *onions_seen_this_period;
+} hs_stats_t;
+
+/** Our statistics structure singleton. */
+static hs_stats_t *hs_stats = NULL;
+
+/** Allocate, initialize and return an hs_stats_t structure. */
+static hs_stats_t *
+hs_stats_new(void)
+{
+  hs_stats_t *new_hs_stats = tor_malloc_zero(sizeof(hs_stats_t));
+  new_hs_stats->onions_seen_this_period = digestmap_new();
+
+  return new_hs_stats;
+}
+
+#define hs_stats_free(val) \
+  FREE_AND_NULL(hs_stats_t, hs_stats_free_, (val))
+
+/** Free an hs_stats_t structure. */
+static void
+hs_stats_free_(hs_stats_t *victim_hs_stats)
+{
+  if (!victim_hs_stats) {
+    return;
+  }
+
+  digestmap_free(victim_hs_stats->onions_seen_this_period, NULL);
+  tor_free(victim_hs_stats);
+}
+
+/** Initialize hidden service statistics. */
+void
+rep_hist_hs_stats_init(time_t now)
+{
+  if (!hs_stats) {
+    hs_stats = hs_stats_new();
+  }
+
+  start_of_hs_stats_interval = now;
+}
+
+/** Clear history of hidden service statistics and set the measurement
+ * interval start to <b>now</b>. */
+static void
+rep_hist_reset_hs_stats(time_t now)
+{
+  if (!hs_stats) {
+    hs_stats = hs_stats_new();
+  }
+
+  hs_stats->rp_relay_cells_seen = 0;
+
+  digestmap_free(hs_stats->onions_seen_this_period, NULL);
+  hs_stats->onions_seen_this_period = digestmap_new();
+
+  start_of_hs_stats_interval = now;
+}
+
+/** Stop collecting hidden service stats in a way that we can re-start
+ * doing so in rep_hist_buffer_stats_init(). */
+void
+rep_hist_hs_stats_term(void)
+{
+  rep_hist_reset_hs_stats(0);
+}
+
+/** We saw a new HS relay cell, Count it! */
+void
+rep_hist_seen_new_rp_cell(void)
+{
+  if (!hs_stats) {
+    return; // We're not collecting stats
+  }
+
+  hs_stats->rp_relay_cells_seen++;
+}
+
+/** As HSDirs, we saw another hidden service with public key
+ *  <b>pubkey</b>. Check whether we have counted it before, if not
+ *  count it now! */
+void
+rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey)
+{
+  char pubkey_hash[DIGEST_LEN];
+
+  if (!hs_stats) {
+    return; // We're not collecting stats
+  }
+
+  /* Get the digest of the pubkey which will be used to detect whether
+     we've seen this hidden service before or not.  */
+  if (crypto_pk_get_digest(pubkey, pubkey_hash) < 0) {
+    /*  This fail should not happen; key has been validated by
+        descriptor parsing code first. */
+    return;
+  }
+
+  /* Check if this is the first time we've seen this hidden
+     service. If it is, count it as new. */
+  if (!digestmap_get(hs_stats->onions_seen_this_period,
+                     pubkey_hash)) {
+    digestmap_set(hs_stats->onions_seen_this_period,
+                  pubkey_hash, (void*)(uintptr_t)1);
+  }
+}
+
+/* The number of cells that are supposed to be hidden from the adversary
+ * by adding noise from the Laplace distribution.  This value, divided by
+ * EPSILON, is Laplace parameter b. It must be greather than 0. */
+#define REND_CELLS_DELTA_F 2048
+/* Security parameter for obfuscating number of cells with a value between
+ * ]0.0, 1.0]. Smaller values obfuscate observations more, but at the same
+ * time make statistics less usable. */
+#define REND_CELLS_EPSILON 0.3
+/* The number of cells that are supposed to be hidden from the adversary
+ * by rounding up to the next multiple of this number. */
+#define REND_CELLS_BIN_SIZE 1024
+/* The number of service identities that are supposed to be hidden from the
+ * adversary by adding noise from the Laplace distribution. This value,
+ * divided by EPSILON, is Laplace parameter b. It must be greater than 0. */
+#define ONIONS_SEEN_DELTA_F 8
+/* Security parameter for obfuscating number of service identities with a
+ * value between ]0.0, 1.0]. Smaller values obfuscate observations more, but
+ * at the same time make statistics less usable. */
+#define ONIONS_SEEN_EPSILON 0.3
+/* The number of service identities that are supposed to be hidden from
+ * the adversary by rounding up to the next multiple of this number. */
+#define ONIONS_SEEN_BIN_SIZE 8
+
+/** Allocate and return a string containing hidden service stats that
+ *  are meant to be placed in the extra-info descriptor. */
+static char *
+rep_hist_format_hs_stats(time_t now)
+{
+  char t[ISO_TIME_LEN+1];
+  char *hs_stats_string;
+  int64_t obfuscated_cells_seen;
+  int64_t obfuscated_onions_seen;
+
+  uint64_t rounded_cells_seen
+    = round_uint64_to_next_multiple_of(hs_stats->rp_relay_cells_seen,
+                                       REND_CELLS_BIN_SIZE);
+  rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX);
+  obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen,
+                          crypto_rand_double(),
+                          REND_CELLS_DELTA_F, REND_CELLS_EPSILON);
+
+  uint64_t rounded_onions_seen =
+    round_uint64_to_next_multiple_of((size_t)digestmap_size(
+                                        hs_stats->onions_seen_this_period),
+                                     ONIONS_SEEN_BIN_SIZE);
+  rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX);
+  obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen,
+                           crypto_rand_double(), ONIONS_SEEN_DELTA_F,
+                           ONIONS_SEEN_EPSILON);
+
+  format_iso_time(t, now);
+  tor_asprintf(&hs_stats_string, "hidserv-stats-end %s (%d s)\n"
+               "hidserv-rend-relayed-cells %"PRId64" delta_f=%d "
+                                           "epsilon=%.2f bin_size=%d\n"
+               "hidserv-dir-onions-seen %"PRId64" delta_f=%d "
+                                        "epsilon=%.2f bin_size=%d\n",
+               t, (unsigned) (now - start_of_hs_stats_interval),
+               (obfuscated_cells_seen), REND_CELLS_DELTA_F,
+               REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE,
+               (obfuscated_onions_seen),
+               ONIONS_SEEN_DELTA_F,
+               ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE);
+
+  return hs_stats_string;
+}
+
+/** If 24 hours have passed since the beginning of the current HS
+ * stats period, write buffer stats to $DATADIR/stats/hidserv-stats
+ * (possibly overwriting an existing file) and reset counters.  Return
+ * when we would next want to write buffer stats or 0 if we never want to
+ * write. */
+time_t
+rep_hist_hs_stats_write(time_t now)
+{
+  char *str = NULL;
+
+  if (!start_of_hs_stats_interval) {
+    return 0; /* Not initialized. */
+  }
+
+  if (start_of_hs_stats_interval + WRITE_STATS_INTERVAL > now) {
+    goto done; /* Not ready to write */
+  }
+
+  /* Generate history string. */
+  str = rep_hist_format_hs_stats(now);
+
+  /* Reset HS history. */
+  rep_hist_reset_hs_stats(now);
+
+  /* Try to write to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "hidserv-stats", str,
+                         "hidden service stats");
+  }
+
+ done:
+  tor_free(str);
+  return start_of_hs_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+static uint64_t link_proto_count[MAX_LINK_PROTO+1][2];
+
+/** Note that we negotiated link protocol version <b>link_proto</b>, on
+ * a connection that started here iff <b>started_here</b> is true.
+ */
+void
+rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
+{
+  started_here = !!started_here; /* force to 0 or 1 */
+  if (link_proto > MAX_LINK_PROTO) {
+    log_warn(LD_BUG, "Can't log link protocol %u", link_proto);
+    return;
+  }
+
+  link_proto_count[link_proto][started_here]++;
+}
+
+/**
+ * Update the maximum count of total pending channel padding timers
+ * in this period.
+ */
+void
+rep_hist_padding_count_timers(uint64_t num_timers)
+{
+  if (num_timers > padding_current.maximum_chanpad_timers) {
+    padding_current.maximum_chanpad_timers = num_timers;
+  }
+}
+
+/**
+ * Count a cell that we sent for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_write(padding_type_t type)
+{
+  switch (type) {
+    case PADDING_TYPE_DROP:
+      padding_current.write_drop_cell_count++;
+      break;
+    case PADDING_TYPE_CELL:
+      padding_current.write_pad_cell_count++;
+      break;
+    case PADDING_TYPE_TOTAL:
+      padding_current.write_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_TOTAL:
+      padding_current.enabled_write_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_CELL:
+      padding_current.enabled_write_pad_cell_count++;
+      break;
+  }
+}
+
+/**
+ * Count a cell that we've received for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_read(padding_type_t type)
+{
+  switch (type) {
+    case PADDING_TYPE_DROP:
+      padding_current.read_drop_cell_count++;
+      break;
+    case PADDING_TYPE_CELL:
+      padding_current.read_pad_cell_count++;
+      break;
+    case PADDING_TYPE_TOTAL:
+      padding_current.read_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_TOTAL:
+      padding_current.enabled_read_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_CELL:
+      padding_current.enabled_read_pad_cell_count++;
+      break;
+  }
+}
+
+/**
+ * Reset our current padding statistics. Called once every 24 hours.
+ */
+void
+rep_hist_reset_padding_counts(void)
+{
+  memset(&padding_current, 0, sizeof(padding_current));
+}
+
+/**
+ * Copy our current cell counts into a structure for listing in our
+ * extra-info descriptor. Also perform appropriate rounding and redaction.
+ *
+ * This function is called once every 24 hours.
+ */
+#define MIN_CELL_COUNTS_TO_PUBLISH 1
+#define ROUND_CELL_COUNTS_TO 10000
+void
+rep_hist_prep_published_padding_counts(time_t now)
+{
+  memcpy(&padding_published, &padding_current, sizeof(padding_published));
+
+  if (padding_published.read_cell_count < MIN_CELL_COUNTS_TO_PUBLISH ||
+      padding_published.write_cell_count < MIN_CELL_COUNTS_TO_PUBLISH) {
+    memset(&padding_published, 0, sizeof(padding_published));
+    return;
+  }
+
+  format_iso_time(padding_published.first_published_at, now);
+#define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \
+                                      ROUND_CELL_COUNTS_TO)
+  ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.read_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count);
+#undef ROUND_AND_SET_COUNT
+}
+
+/**
+ * Returns an allocated string for extra-info documents for publishing
+ * padding statistics from the last 24 hour interval.
+ */
+char *
+rep_hist_get_padding_count_lines(void)
+{
+  char *result = NULL;
+
+  if (!padding_published.read_cell_count ||
+          !padding_published.write_cell_count) {
+    return NULL;
+  }
+
+  tor_asprintf(&result, "padding-counts %s (%d s)"
+                        " bin-size=%"PRIu64
+                        " write-drop=%"PRIu64
+                        " write-pad=%"PRIu64
+                        " write-total=%"PRIu64
+                        " read-drop=%"PRIu64
+                        " read-pad=%"PRIu64
+                        " read-total=%"PRIu64
+                        " enabled-read-pad=%"PRIu64
+                        " enabled-read-total=%"PRIu64
+                        " enabled-write-pad=%"PRIu64
+                        " enabled-write-total=%"PRIu64
+                        " max-chanpad-timers=%"PRIu64
+                        "\n",
+               padding_published.first_published_at,
+               REPHIST_CELL_PADDING_COUNTS_INTERVAL,
+               (uint64_t)ROUND_CELL_COUNTS_TO,
+               (padding_published.write_drop_cell_count),
+               (padding_published.write_pad_cell_count),
+               (padding_published.write_cell_count),
+               (padding_published.read_drop_cell_count),
+               (padding_published.read_pad_cell_count),
+               (padding_published.read_cell_count),
+               (padding_published.enabled_read_pad_cell_count),
+               (padding_published.enabled_read_cell_count),
+               (padding_published.enabled_write_pad_cell_count),
+               (padding_published.enabled_write_cell_count),
+               (padding_published.maximum_chanpad_timers)
+               );
+
+  return result;
+}
+
+/** Log a heartbeat message explaining how many connections of each link
+ * protocol version we have used.
+ */
+void
+rep_hist_log_link_protocol_counts(void)
+{
+  log_notice(LD_HEARTBEAT,
+             "Since startup, we have initiated "
+             "%"PRIu64" v1 connections, "
+             "%"PRIu64" v2 connections, "
+             "%"PRIu64" v3 connections, and "
+             "%"PRIu64" v4 connections; and received "
+             "%"PRIu64" v1 connections, "
+             "%"PRIu64" v2 connections, "
+             "%"PRIu64" v3 connections, and "
+             "%"PRIu64" v4 connections.",
+             (link_proto_count[1][1]),
+             (link_proto_count[2][1]),
+             (link_proto_count[3][1]),
+             (link_proto_count[4][1]),
+             (link_proto_count[1][0]),
+             (link_proto_count[2][0]),
+             (link_proto_count[3][0]),
+             (link_proto_count[4][0]));
+}
+
+/** Free all storage held by the OR/link history caches, by the
+ * bandwidth history arrays, by the port history, or by statistics . */
+void
+rep_hist_free_all(void)
+{
+  hs_stats_free(hs_stats);
+  digestmap_free(history_map, free_or_history);
+
+  bw_array_free(read_array);
+  read_array = NULL;
+
+  bw_array_free(write_array);
+  write_array = NULL;
+
+  bw_array_free(dir_read_array);
+  dir_read_array = NULL;
+
+  bw_array_free(dir_write_array);
+  dir_write_array = NULL;
+
+  tor_free(exit_bytes_read);
+  tor_free(exit_bytes_written);
+  tor_free(exit_streams);
+  predicted_ports_free_all();
+  bidi_map_free_all();
+
+  if (circuits_for_buffer_stats) {
+    SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, s,
+                      tor_free(s));
+    smartlist_free(circuits_for_buffer_stats);
+    circuits_for_buffer_stats = NULL;
+  }
+  rep_hist_desc_stats_term();
+  total_descriptor_downloads = 0;
+
+  tor_assert_nonfatal(rephist_total_alloc == 0);
+  tor_assert_nonfatal_once(rephist_total_num == 0);
+}
diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h
new file mode 100644
index 0000000000..06a5e48211
--- /dev/null
+++ b/src/feature/stats/rephist.h
@@ -0,0 +1,140 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rephist.h
+ * \brief Header file for rephist.c.
+ **/
+
+#ifndef TOR_REPHIST_H
+#define TOR_REPHIST_H
+
+void rep_hist_init(void);
+void rep_hist_dump_stats(time_t now, int severity);
+void rep_hist_note_bytes_read(size_t num_bytes, time_t when);
+void rep_hist_note_bytes_written(size_t num_bytes, time_t when);
+
+void rep_hist_make_router_pessimal(const char *id, time_t when);
+
+void rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when);
+void rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when);
+
+int rep_hist_bandwidth_assess(void);
+char *rep_hist_get_bandwidth_lines(void);
+void rep_hist_update_state(or_state_t *state);
+int rep_hist_load_state(or_state_t *state, char **err);
+void rep_history_clean(time_t before);
+
+void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr,
+                                    const uint16_t at_port, time_t when);
+void rep_hist_note_router_unreachable(const char *id, time_t when);
+int rep_hist_record_mtbf_data(time_t now, int missing_means_down);
+int rep_hist_load_mtbf_data(time_t now);
+
+time_t rep_hist_downrate_old_runs(time_t now);
+long rep_hist_get_uptime(const char *id, time_t when);
+double rep_hist_get_stability(const char *id, time_t when);
+double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when);
+long rep_hist_get_weighted_time_known(const char *id, time_t when);
+int rep_hist_have_measured_enough_stability(void);
+
+void predicted_ports_init(void);
+void rep_hist_note_used_port(time_t now, uint16_t port);
+smartlist_t *rep_hist_get_predicted_ports(time_t now);
+void rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports);
+void rep_hist_note_used_resolve(time_t now);
+void rep_hist_note_used_internal(time_t now, int need_uptime,
+                                 int need_capacity);
+int rep_hist_get_predicted_internal(time_t now, int *need_uptime,
+                                    int *need_capacity);
+
+int any_predicted_circuits(time_t now);
+int rep_hist_circbuilding_dormant(time_t now);
+int predicted_ports_prediction_time_remaining(time_t now);
+
+void rep_hist_exit_stats_init(time_t now);
+void rep_hist_reset_exit_stats(time_t now);
+void rep_hist_exit_stats_term(void);
+char *rep_hist_format_exit_stats(time_t now);
+time_t rep_hist_exit_stats_write(time_t now);
+void rep_hist_note_exit_bytes(uint16_t port, size_t num_written,
+                              size_t num_read);
+void rep_hist_note_exit_stream_opened(uint16_t port);
+
+void rep_hist_buffer_stats_init(time_t now);
+void rep_hist_buffer_stats_add_circ(circuit_t *circ,
+                                    time_t end_of_interval);
+time_t rep_hist_buffer_stats_write(time_t now);
+void rep_hist_buffer_stats_term(void);
+void rep_hist_add_buffer_stats(double mean_num_cells_in_queue,
+     double mean_time_cells_in_queue, uint32_t processed_cells);
+char *rep_hist_format_buffer_stats(time_t now);
+void rep_hist_reset_buffer_stats(time_t now);
+
+void rep_hist_desc_stats_init(time_t now);
+void rep_hist_note_desc_served(const char * desc);
+void rep_hist_desc_stats_term(void);
+time_t rep_hist_desc_stats_write(time_t now);
+
+void rep_hist_conn_stats_init(time_t now);
+void rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read,
+                                 size_t num_written, time_t when);
+void rep_hist_reset_conn_stats(time_t now);
+char *rep_hist_format_conn_stats(time_t now);
+time_t rep_hist_conn_stats_write(time_t now);
+void rep_hist_conn_stats_term(void);
+
+void rep_hist_note_circuit_handshake_requested(uint16_t type);
+void rep_hist_note_circuit_handshake_assigned(uint16_t type);
+void rep_hist_log_circuit_handshake_stats(time_t now);
+
+void rep_hist_hs_stats_init(time_t now);
+void rep_hist_hs_stats_term(void);
+time_t rep_hist_hs_stats_write(time_t now);
+char *rep_hist_get_hs_stats_string(void);
+void rep_hist_seen_new_rp_cell(void);
+void rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey);
+
+void rep_hist_free_all(void);
+
+void rep_hist_note_negotiated_link_proto(unsigned link_proto,
+                                         int started_here);
+void rep_hist_log_link_protocol_counts(void);
+
+extern uint64_t rephist_total_alloc;
+extern uint32_t rephist_total_num;
+#ifdef TOR_UNIT_TESTS
+extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1];
+extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1];
+#endif
+
+/**
+ * Represents the type of a cell for padding accounting
+ */
+typedef enum padding_type_t {
+    /** A RELAY_DROP cell */
+    PADDING_TYPE_DROP,
+    /** A CELL_PADDING cell */
+    PADDING_TYPE_CELL,
+    /** Total counts of padding and non-padding together */
+    PADDING_TYPE_TOTAL,
+    /** Total cell counts for all padding-enabled channels */
+    PADDING_TYPE_ENABLED_TOTAL,
+    /** CELL_PADDING counts for all padding-enabled channels */
+    PADDING_TYPE_ENABLED_CELL
+} padding_type_t;
+
+/** The amount of time over which the padding cell counts were counted */
+#define REPHIST_CELL_PADDING_COUNTS_INTERVAL (24*60*60)
+void rep_hist_padding_count_read(padding_type_t type);
+void rep_hist_padding_count_write(padding_type_t type);
+char *rep_hist_get_padding_count_lines(void);
+void rep_hist_reset_padding_counts(void);
+void rep_hist_prep_published_padding_counts(time_t now);
+void rep_hist_padding_count_timers(uint64_t num_timers);
+
+#endif /* !defined(TOR_REPHIST_H) */
+